[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_226":227,"_228":229},"routes/_app.x.$handle.$name",{"_96":95,"_32":45,"_104":86,"_27":105,"_13":14,"_106":-5,"_11":107,"_108":109,"_110":111,"_112":113,"_114":115,"_20":-5,"_116":-5,"_117":-5,"_118":25,"_119":120,"_121":25,"_122":-5,"_123":-5,"_124":125},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":55,"_18":56,"_57":-5,"_58":25,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":25},"version",17,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":37,"_35":36,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.ts","latestPath","fileId","0199d8a3-6d60-729d-ba23-eb6ecd94adf1","name","fileType","script","content","import { fetchEngagedUsers } from \"./github.ts\";\nimport { sendToClay } from \"./clay.ts\";\nimport {\n GITHUB_REPO,\n ISSUE_LIMIT,\n MAX_STARGAZERS,\n MAX_FORKS,\n MAX_COMMENTS_PER_ISSUE,\n FETCH_REACTIONS,\n ENABLE_DEDUPLICATION,\n} from \"./config.ts\";\nimport {\n initDatabase,\n isUserTracked,\n addTrackedUsers,\n getStats,\n} from \"./database.ts\";\n\nexport default async function () {\n // Make sure to set your Clay Webhook URL in the Environment Variables!\n const clayWebhookUrl = Deno.env.get(\"CLAY_WEBHOOK_URL\");\n\n if (!clayWebhookUrl) {\n throw new Error(\"CLAY_WEBHOOK_URL environment variable is required\");\n }\n\n const [owner, repo] = String(GITHUB_REPO).split(\"/\");\n\n // Initialize database for user tracking\n if (ENABLE_DEDUPLICATION) {\n await initDatabase();\n console.log(\"Database initialized for user deduplication\");\n }\n\n // Fetch engaged users\n const engagedUsernames = await fetchEngagedUsers(owner, repo, {\n issueLimit: ISSUE_LIMIT,\n maxStargazers: MAX_STARGAZERS,\n maxForks: MAX_FORKS,\n maxCommentsPerIssue: MAX_COMMENTS_PER_ISSUE,\n fetchReactions: FETCH_REACTIONS,\n });\n console.log(`Found ${engagedUsernames.length} engaged users`);\n\n // Filter out already-tracked users (if deduplication is enabled)\n let newUsernames = engagedUsernames;\n if (ENABLE_DEDUPLICATION) {\n newUsernames = [];\n for (const username of engagedUsernames) {\n const isTracked = await isUserTracked(username);\n if (!isTracked) {\n newUsernames.push(username);\n }\n }\n const alreadyTracked = engagedUsernames.length - newUsernames.length;\n console.log(\n `Filtered users: ${newUsernames.length} new, ${alreadyTracked} already tracked`,\n );\n\n // Show database stats\n const stats = await getStats();\n console.log(\n `Database stats: ${stats.totalUsers} total users tracked across all repos`,\n );\n }\n\n // Format data for Clay integration - one record per user\n const userData = newUsernames.map((username) => ({\n github_username: username,\n source_repo: GITHUB_REPO,\n source: \"github_to_clay\",\n }));\n\n // Log data\n if (userData.length > 0 && userData.length <= 10) {\n console.log(`\\n=== User Data (for Clay integration) ===`);\n userData.forEach((record) => {\n console.log(JSON.stringify(record, null, 2));\n });\n } else if (userData.length > 10) {\n console.log(`\\n=== Sample User Data (showing first 3 of ${userData.length}) ===`);\n userData.slice(0, 3).forEach((record) => {\n console.log(JSON.stringify(record, null, 2));\n });\n }\n\n console.log(`\\n→ Sending ${userData.length} user records to Clay...`);\n\n // Send to Clay\n if (userData.length > 0) {\n await sendToClay(clayWebhookUrl, userData);\n console.log(`✓ Successfully sent ${userData.length} records to Clay`);\n\n // Add newly sent users to the tracking database\n if (ENABLE_DEDUPLICATION) {\n await addTrackedUsers(\n newUsernames.map((username) => ({\n username,\n sourceRepo: GITHUB_REPO,\n })),\n );\n console.log(`✓ Added ${newUsernames.length} users to tracking database`);\n }\n } else {\n console.log(`⚠ No new user records to send`);\n }\n\n console.log(`\\n✓ Cron job completed successfully`);\n}\n","interval","createdAt",["D",1760377185915],"branchVersion","branchId","835fc6d8-a770-11f0-b7ff-0224a6c84d84","id","835c1240-a770-11f0-b7ff-0224a6c84d84","GitHub-To-Clay",["D",1760276213018],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Scrape users from any GitHub repo, and ingest into Clay","imageUrl","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/7d1bd1e4-9009-406c-8bf6-a238d334e100/square512?exp=1789570799&sig=0b9cd25d0b472bb71174dd689e856a70ba149299a1764162201fae3e0b5a0fcf","0199ca5b-d33c-7148-89fe-6858408dafe2","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","0199ca5b-d33c-7148-89fe-6f6326904a93","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_96":95,"_103":99},"org",{"_96":95,"_97":-5,"_98":99,"_18":56,"_100":101,"_102":25},"branch",{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"isOwned",28,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","whp","handle","displayName","avatar","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/0dc740ff-fa57-445c-3eec-80fd20daa500/square512?exp=1789570799&sig=bf7009496a93558c3cf09ef626f7b321f228de5647da8f3842ff3ad59943417b","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":222,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":55,"_18":56,"_57":-5,"_58":25,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":223,"_80":224,"_82":225,"_84":25},"tree",[193,194,195,196,197,198,199],"files",[193,194,195,196,197,198,199],"counts",{"_182":62,"_183":184,"_185":143,"_186":187,"_188":189},"contents",[126,127,128,129,130,131,132],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":175,"_32":176,"_30":177,"_134":-5,"_41":42,"_40":178,"_135":151,"_38":179,"_13":62,"_33":20,"_35":180,"_24":25,"_138":-5,"_27":181},{"_43":168,"_32":169,"_30":170,"_134":-5,"_41":42,"_40":171,"_135":25,"_38":172,"_13":137,"_33":20,"_35":173,"_24":25,"_138":-5,"_27":174},{"_43":162,"_32":163,"_30":164,"_134":-5,"_41":42,"_40":143,"_135":25,"_38":165,"_13":62,"_33":34,"_35":166,"_24":25,"_138":-5,"_27":167},{"_43":155,"_32":156,"_30":157,"_134":-5,"_41":42,"_40":158,"_135":25,"_38":159,"_13":143,"_33":34,"_35":160,"_24":25,"_138":-5,"_27":161},{"_43":147,"_32":148,"_30":149,"_134":-5,"_41":42,"_40":150,"_135":151,"_38":152,"_13":62,"_33":34,"_35":153,"_24":25,"_138":-5,"_27":154},{"_43":140,"_32":141,"_30":142,"_134":-5,"_41":42,"_40":143,"_135":25,"_38":144,"_13":62,"_33":34,"_35":145,"_24":25,"_138":-5,"_27":146},{"_43":133,"_32":28,"_30":31,"_134":-5,"_41":42,"_40":14,"_135":25,"_38":136,"_13":137,"_33":37,"_35":36,"_24":25,"_138":-5,"_27":139},"9be1fc62-a85b-11f0-808f-0224a6c84d84","parentId","latest",["D",1760377185915],4,"label",[28],"8362d86e-a770-11f0-b7ff-0224a6c84d84","github.ts","0199d8a3-6d60-729d-ba23-e76c06e5f7ee",1,["D",1760276213018],"import { fetch } from \"https://esm.town/v/std/fetch\";\n\nexport interface EngagedUser {\n github_username: string;\n source_repo: string;\n}\n\nexport async function fetchAllIssues(\n owner: string,\n repo: string,\n limit?: number,\n): Promise {\n const issues: any[] = [];\n let page = 1;\n const perPage = limit && limit < 100 ? limit : 100;\n\n const headers: Record = {\n Accept: \"application/vnd.github.v3+json\",\n \"User-Agent\": \"GitHubEngagementTracker/1.0\",\n };\n\n while (true) {\n const remainder = limit\n ? Math.min(perPage, limit - issues.length)\n : perPage;\n\n if (remainder <= 0) break;\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/issues?page=${page}&per_page=${remainder}&state=all`,\n { headers },\n );\n\n if (!response.ok) {\n throw new Error(\n `GitHub API error: ${response.status} ${response.statusText}`,\n );\n }\n\n const data = await response.json();\n\n const filteredIssues = data\n .filter((item) => !item.pull_request)\n .map((item) => ({\n number: item.number,\n title: item.title,\n body: item.body || \"\",\n author_username: item.user.login,\n created_at: item.created_at,\n updated_at: item.updated_at,\n state: item.state,\n labels: item.labels.map((label) => label.name),\n html_url: item.html_url,\n }));\n\n issues.push(...filteredIssues);\n\n if (data.length < remainder) break;\n\n page++;\n }\n\n return issues;\n}\n\nexport async function fetchIssueComments(\n owner: string,\n repo: string,\n issueNumber: number,\n limit?: number,\n): Promise {\n const comments: any[] = [];\n let page = 1;\n const perPage = limit && limit < 100 ? limit : 100;\n\n const headers: Record = {\n Accept: \"application/vnd.github.v3+json\",\n \"User-Agent\": \"GitHubEngagementTracker/1.0\",\n };\n\n while (true) {\n const remainder = limit\n ? Math.min(perPage, limit - comments.length)\n : perPage;\n\n if (remainder <= 0) break;\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments?page=${page}&per_page=${remainder}`,\n { headers },\n );\n\n if (!response.ok) {\n if (response.status === 404) return comments;\n throw new Error(\n `GitHub API error fetching comments: ${response.status} ${response.statusText}`,\n );\n }\n\n const data = await response.json();\n\n const mappedComments = data.map((item: any) => ({\n id: item.id,\n body: item.body || \"\",\n author_username: item.user.login,\n created_at: item.created_at,\n issue_number: issueNumber,\n }));\n\n comments.push(...mappedComments);\n\n if (data.length < remainder) break;\n page++;\n }\n\n return comments;\n}\n\nexport async function fetchIssueReactions(\n owner: string,\n repo: string,\n issueNumber: number,\n): Promise {\n const headers: Record = {\n Accept: \"application/vnd.github.v3+json\",\n \"User-Agent\": \"GitHubEngagementTracker/1.0\",\n };\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/reactions`,\n { headers },\n );\n\n if (!response.ok) {\n if (response.status === 404) return [];\n throw new Error(\n `GitHub API error fetching issue reactions: ${response.status} ${response.statusText}`,\n );\n }\n\n const data = await response.json();\n\n return data.map((item: any) => ({\n content: item.content,\n user_username: item.user.login,\n created_at: item.created_at,\n }));\n}\n\nexport async function fetchCommentReactions(\n owner: string,\n repo: string,\n commentId: number,\n): Promise {\n const headers: Record = {\n Accept: \"application/vnd.github.v3+json\",\n \"User-Agent\": \"GitHubEngagementTracker/1.0\",\n };\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/issues/comments/${commentId}/reactions`,\n { headers },\n );\n\n if (!response.ok) {\n if (response.status === 404) return [];\n throw new Error(\n `GitHub API error fetching comment reactions: ${response.status} ${response.statusText}`,\n );\n }\n\n const data = await response.json();\n\n return data.map((item: any) => ({\n content: item.content,\n user_username: item.user.login,\n created_at: item.created_at,\n }));\n}\n\nexport async function fetchStargazers(\n owner: string,\n repo: string,\n limit?: number,\n): Promise {\n const stargazers: string[] = [];\n let page = 1;\n const perPage = limit && limit < 100 ? limit : 100;\n\n const headers: Record = {\n Accept: \"application/vnd.github.v3+json\",\n \"User-Agent\": \"GitHubEngagementTracker/1.0\",\n };\n\n while (true) {\n const remainder = limit\n ? Math.min(perPage, limit - stargazers.length)\n : perPage;\n\n if (remainder <= 0) break;\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/stargazers?page=${page}&per_page=${remainder}`,\n { headers },\n );\n\n if (!response.ok) {\n if (response.status === 404) return stargazers;\n throw new Error(\n `GitHub API error fetching stargazers: ${response.status} ${response.statusText}`,\n );\n }\n\n const data = await response.json();\n\n for (const item of data) {\n stargazers.push(item.login);\n }\n\n if (data.length < remainder) break;\n page++;\n }\n\n return stargazers;\n}\n\nexport async function fetchForks(\n owner: string,\n repo: string,\n limit?: number,\n): Promise {\n const forkers: string[] = [];\n let page = 1;\n const perPage = limit && limit < 100 ? limit : 100;\n\n const headers: Record = {\n Accept: \"application/vnd.github.v3+json\",\n \"User-Agent\": \"GitHubEngagementTracker/1.0\",\n };\n\n while (true) {\n const remainder = limit\n ? Math.min(perPage, limit - forkers.length)\n : perPage;\n\n if (remainder <= 0) break;\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/forks?page=${page}&per_page=${remainder}`,\n { headers },\n );\n\n if (!response.ok) {\n if (response.status === 404) return forkers;\n throw new Error(\n `GitHub API error fetching forks: ${response.status} ${response.statusText}`,\n );\n }\n\n const data = await response.json();\n\n for (const item of data) {\n forkers.push(item.owner.login);\n }\n\n if (data.length < remainder) break;\n page++;\n }\n\n return forkers;\n}\n\ninterface FetchConfig {\n issueLimit?: number;\n maxStargazers?: number;\n maxForks?: number;\n maxCommentsPerIssue?: number;\n fetchReactions: boolean;\n}\n\nexport async function fetchEngagedUsers(\n owner: string,\n repo: string,\n config: FetchConfig,\n): Promise {\n console.log(`Fetching engaged users for ${owner}/${repo}...`);\n\n const issues = await fetchAllIssues(owner, repo, config.issueLimit);\n console.log(`Found ${issues.length} issues`);\n\n const engagedUsers = new Set();\n\n // Collect issue creators\n for (const issue of issues) {\n engagedUsers.add(issue.author_username);\n }\n\n // Fetch comments and reactions for each issue\n for (const issue of issues) {\n // Fetch issue comments\n try {\n const comments = await fetchIssueComments(\n owner,\n repo,\n issue.number,\n config.maxCommentsPerIssue,\n );\n\n for (const comment of comments) {\n engagedUsers.add(comment.author_username);\n\n // Fetch reactions for this comment (if enabled)\n if (config.fetchReactions) {\n try {\n const commentReactions = await fetchCommentReactions(\n owner,\n repo,\n comment.id,\n );\n\n for (const reaction of commentReactions) {\n engagedUsers.add(reaction.user_username);\n }\n } catch (error) {\n console.log(\n ` Error fetching reactions for comment ${comment.id}:`,\n error.message,\n );\n }\n }\n }\n } catch (error) {\n console.log(\n ` Error fetching comments for issue #${issue.number}:`,\n error.message,\n );\n }\n\n // Fetch issue reactions (if enabled)\n if (config.fetchReactions) {\n try {\n const issueReactions = await fetchIssueReactions(\n owner,\n repo,\n issue.number,\n );\n\n for (const reaction of issueReactions) {\n engagedUsers.add(reaction.user_username);\n }\n } catch (error) {\n console.log(\n ` Error fetching reactions for issue #${issue.number}:`,\n error.message,\n );\n }\n }\n }\n\n // Fetch stargazers\n try {\n const stargazers = await fetchStargazers(owner, repo, config.maxStargazers);\n console.log(`Found ${stargazers.length} stargazers`);\n for (const username of stargazers) {\n engagedUsers.add(username);\n }\n } catch (error) {\n console.log(` Error fetching stargazers:`, error.message);\n }\n\n // Fetch forkers\n try {\n const forkers = await fetchForks(owner, repo, config.maxForks);\n console.log(`Found ${forkers.length} forkers`);\n for (const username of forkers) {\n engagedUsers.add(username);\n }\n } catch (error) {\n console.log(` Error fetching forkers:`, error.message);\n }\n\n console.log(`Total unique engaged users: ${engagedUsers.size}`);\n return Array.from(engagedUsers);\n}\n",[141],"6d6c792e-a839-11f0-b62f-0224a6c84d84","database.ts","6d6c7a00-a839-11f0-b62f-0224a6c84d84",7,true,["D",1760362505083],"import { sqlite } from \"https://esm.town/v/stevekrouse/sqlite\";\n\nconst TABLE_NAME = \"tracked_users\";\n\n/**\n * Initialize the database and create the tracked_users table if it doesn't exist\n */\nexport async function initDatabase(): Promise {\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (\n username TEXT PRIMARY KEY,\n source_repo TEXT NOT NULL,\n first_seen TEXT NOT NULL\n )\n `);\n}\n\n/**\n * Check if a user is already tracked in the database\n */\nexport async function isUserTracked(username: string): Promise {\n const result = await sqlite.execute(\n `SELECT 1 FROM ${TABLE_NAME} WHERE username = ? LIMIT 1`,\n [username],\n );\n return result.rows.length > 0;\n}\n\n/**\n * Add multiple users to the tracked users database\n */\nexport async function addTrackedUsers(\n users: Array<{ username: string; sourceRepo: string }>,\n): Promise {\n if (users.length === 0) return;\n\n const timestamp = new Date().toISOString();\n\n // Use a transaction for bulk insert\n for (const user of users) {\n await sqlite.execute(\n `INSERT OR IGNORE INTO ${TABLE_NAME} (username, source_repo, first_seen) VALUES (?, ?, ?)`,\n [user.username, user.sourceRepo, timestamp],\n );\n }\n}\n\n/**\n * Get statistics about tracked users\n */\nexport async function getStats(): Promise<{\n totalUsers: number;\n byRepo: Record;\n}> {\n const totalResult = await sqlite.execute(\n `SELECT COUNT(*) as count FROM ${TABLE_NAME}`,\n );\n const totalUsers = totalResult.rows[0]?.count || 0;\n\n const repoResult = await sqlite.execute(\n `SELECT source_repo, COUNT(*) as count FROM ${TABLE_NAME} GROUP BY source_repo`,\n );\n\n const byRepo: Record = {};\n for (const row of repoResult.rows) {\n byRepo[row.source_repo] = row.count;\n }\n\n return { totalUsers, byRepo };\n}\n",[148],"21e0955c-a858-11f0-8270-0224a6c84d84","config.ts","6d72c4c8-a839-11f0-a827-0224a6c84d84",16,["D",1760375692732],"// GitHub repository to track (format: \"owner/repo\")\nexport const GITHUB_REPO = \"val-town/vt\";\n\n// Optional: Limit number of issues to fetch (undefined = fetch all)\nexport const ISSUE_LIMIT = 10 as number | undefined;\n\n// Optional: Limit number of stargazers to fetch (undefined = fetch all)\nexport const MAX_STARGAZERS = 20 as number | undefined;\n\n// Optional: Limit number of forks to fetch (undefined = fetch all)\nexport const MAX_FORKS = 10 as number | undefined;\n\n// Optional: Limit comments per issue (undefined = fetch all)\nexport const MAX_COMMENTS_PER_ISSUE = 5 as number | undefined;\n\n// Optional: Skip fetching reactions (faster processing)\nexport const FETCH_REACTIONS = true;\n\n// Optional: Enable user deduplication to prevent sending duplicate users to Clay\n// When enabled, users are tracked in a SQLite database and only new users are sent\nexport const ENABLE_DEDUPLICATION = true;\n",[156],"8362d60c-a770-11f0-b7ff-0224a6c84d84","clay.ts","0199d8a3-6d60-729d-ba23-e17b57d26bb6",["D",1760276213018],"export async function sendToClay(\n webhookUrl: string,\n data: any[],\n): Promise {\n for (const record of data) {\n const response = await fetch(webhookUrl, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(record),\n });\n\n if (!response.ok) {\n const errorText = await response.text();\n throw new Error(`Clay request failed: ${response.status} ${errorText}`);\n }\n }\n}\n",[163],"2500031c-a84e-11f0-a382-0224a6c84d84","README.md","0199d8a3-6d60-729d-ba23-dd426b995c42",15,["D",1760371403000],"# GitHub to Clay Scraper\n\n![CleanShot_2025-10-08_at_17.41.422x.png](https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/86543734-10b8-431c-e71e-da3da9c11400/public)\nAutomatically ingest GitHub accounts of everyone who interacts with your repo into Clay.\n\nThis val tracks anyone who:\n\n- Created an issue\n- Reacted to an issue\n- Commented on an issue\n- Reacted to an issue's comments\n- Starred a repo\n- Forked a repo\n\n## Setup\n\n1. Click `Remix` on the top-right to get a copy of it\n1. Set up a Clay workbook with a `Webhook` column\n1. Copy your Clay workbook's `Webhook` URL\n ![CleanShot 2025-08-08 at 13.23.47@2x.png](https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/48cd9c4d-2990-480f-9d07-fb09e2282500/public)\n\n1. Set that as `CLAY_WEBHOOK_URL` in this val's `Environment variables` on the\n left sidebar\n1. In [config.ts](./config.ts), replace `GITHUB_REPO` with the full repository name\n (format: `\"owner/repo\"`)\n1. And that's it! The cron will run on your repo every 30 minutes from now on.\n To test it out immediately, navigate to [main.ts](./main.ts) and click `Run`.\n\n## Files\n\n- [config.ts](config.ts) - Configuration settings for the scraper\n- [database.ts](database.ts) - SQLite database for tracking sent users\n- [clay.ts](clay.ts) - Send data to Clay\n- [github.ts](github.ts) - Collects engaged users and fetches usernames\n- [main.ts](main.ts) - Cron trigger that orchestrates the scraping and Clay\n integration\n\n## User Deduplication\n\nThis scraper automatically tracks which users have been sent to Clay to prevent\nduplicates. The feature is controlled by the `ENABLE_DEDUPLICATION` flag in\n[config.ts](./config.ts) (enabled by default).\n\n**How it works:**\n\n- Users are stored in a SQLite database after being successfully sent to Clay\n- On subsequent runs, the scraper checks the database and only sends new users\n- The database persists across all cron runs, so users are only sent once\n- Each user is tracked by their GitHub username and source repository\n\n**Database details:**\n\n- Users are tracked in the `tracked_users` table\n- Stores: username, source repo, and timestamp of first encounter\n- The database automatically initializes on the first run\n\n**To disable deduplication:**\nSet `ENABLE_DEDUPLICATION = false` in [config.ts](./config.ts) to send all\nengaged users on every run (useful for testing or if you want to manage\ndeduplication in Clay instead).\n\n## Rate Limiting\n\nOn larger repos, you may get rate-limited by GitHub. To mitigate this, Val Town\nuses a [proxied fetch](https://docs.val.town/std/fetch/) that reroutes requests\nusing a proxy vendor so that requests obtain different IP addresses. It also\nautomatically retries failed requests several times. Note that Note that using\n`std/fetch` will be significantly slower than directly calling the Javascript\nFetch API due to extra network hops.\n\n",[169],"1aa29808-a84e-11f0-aff1-0224a6c84d84","AGENTS.md","1aa298c6-a84e-11f0-aff1-0224a6c84d84",13,["D",1760371385620],"You are an advanced assistant specialized in generating Val Town code.\n\n## Core Guidelines\n\n- Ask clarifying questions when requirements are ambiguous\n- Provide complete, functional solutions rather than skeleton implementations\n- Test your logic against edge cases before presenting the final solution\n- Ensure all code follows Val Town's specific platform requirements\n- If a section of code that you're working on is getting too complex, consider refactoring it into subcomponents\n\n## Code Standards\n\n- Generate code in TypeScript or TSX\n- Add appropriate TypeScript types and interfaces for all data structures\n- Prefer official SDKs or libraries than writing API calls directly\n- Ask the user to supply API or library documentation if you are at all unsure about it\n- **Never bake in secrets into the code** - always use environment variables\n- Include comments explaining complex logic (avoid commenting obvious operations)\n- Follow modern ES6+ conventions and functional programming practices if possible\n\n## Types of triggers\n\n### 1. HTTP Trigger\n\n- Create web APIs and endpoints\n- Handle HTTP requests and responses\n- Example structure:\n\n```ts\nexport default async function (req: Request) {\n return new Response(\"Hello World\");\n}\n```\n\nFiles that are HTTP triggers have http in their name like `foobar.http.tsx`\n\n### 2. Cron Triggers\n\n- Run on a schedule\n- Use cron expressions for timing\n- Example structure:\n\n```ts\nexport default async function () {\n // Scheduled task code\n}\n```\n\nFiles that are Cron triggers have cron in their name like `foobar.cron.tsx`\n\n### 3. Email Triggers\n\n- Process incoming emails\n- Handle email-based workflows\n- Example structure:\n\n```ts\nexport default async function (email: Email) {\n // Process email\n}\n```\n\nFiles that are Email triggers have email in their name like `foobar.email.tsx`\n\n\n## Val Town Standard Libraries\n\nVal Town provides several hosted services and utility functions.\n\n### Blob Storage\n\n```ts\nimport { blob } from \"https://esm.town/v/std/blob\";\nawait blob.setJSON(\"myKey\", { hello: \"world\" });\nlet blobDemo = await blob.getJSON(\"myKey\");\nlet appKeys = await blob.list(\"app_\");\nawait blob.delete(\"myKey\");\n```\n\n### SQLite\n\n```ts\nimport { sqlite } from \"https://esm.town/v/stevekrouse/sqlite\";\nconst TABLE_NAME = 'todo_app_users_2';\n// Create table - do this before usage and change table name when modifying schema\nawait sqlite.execute(`CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n name TEXT NOT NULL\n)`);\n// Query data\nconst result = await sqlite.execute(`SELECT * FROM ${TABLE_NAME} WHERE id = ?`, [1]);\n```\n\nNote: When changing a SQLite table's schema, change the table's name (e.g., add _2 or _3) to create a fresh table.\n\n### OpenAI\n\n```ts\nimport { OpenAI } from \"https://esm.town/v/std/openai\";\nconst openai = new OpenAI();\nconst completion = await openai.chat.completions.create({\n messages: [\n { role: \"user\", content: \"Say hello in a creative way\" },\n ],\n model: \"gpt-4o-mini\",\n max_tokens: 30,\n});\n```\n\n### Email\n\n```ts\nimport { email } from \"https://esm.town/v/std/email\";\n// By default emails the owner of the val\nawait email({ \n subject: \"Hi\", \n text: \"Hi\", \n html: \"

Hi

\"\n});\n```\n\n## Val Town Utility Functions\n\nVal Town provides several utility functions to help with common project tasks.\n\n### Importing Utilities\n\nAlways import utilities with version pins to avoid breaking changes:\n\n```ts\nimport { parseProject, readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n```\n\n### Available Utilities\n\n\n#### **serveFile** - Serve project files with proper content types\n\nFor example, in Hono:\n\n```ts\n// serve all files in frontend/ and shared/\napp.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\napp.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n```\n\n#### **readFile** - Read files from within the project:\n\n```ts\n// Read a file from the project\nconst fileContent = await readFile(\"/frontend/index.html\", import.meta.url);\n```\n\n#### **listFiles** - List all files in the project\n\n```ts\nconst files = await listFiles(import.meta.url);\n```\n\n#### **parseProject** - Extract information about the current project from import.meta.url\n\nThis is useful for including info for linking back to a val, ie in \"view source\" urls:\n\n```ts\nconst projectVal = parseProject(import.meta.url);\nconsole.log(projectVal.username); // Owner of the project\nconsole.log(projectVal.name); // Project name\nconsole.log(projectVal.version); // Version number\nconsole.log(projectVal.branch); // Branch name\nconsole.log(projectVal.links.self.project); // URL to the project page\n```\n\nHowever, it's *extremely importing* to note that `parseProject` and other Standard Library utilities ONLY RUN ON THE SERVER.\nIf you need access to this data on the client, run it in the server and pass it to the client by splicing it into the HTML page\nor by making an API request for it.\n\n## Val Town Platform Specifics\n\n- **Redirects:** Use `return new Response(null, { status: 302, headers: { Location: \"/place/to/redirect\" }})` instead of `Response.redirect` which is broken\n- **Images:** Avoid external images or base64 images. Use emojis, unicode symbols, or icon fonts/libraries instead\n- **AI Image:** To inline generate an AI image use: `

`\n- **Storage:** DO NOT use the Deno KV module for storage\n- **Browser APIs:** DO NOT use the `alert()`, `prompt()`, or `confirm()` methods\n- **Weather Data:** Use open-meteo for weather data (doesn't require API keys) unless otherwise specified\n- **View Source:** Add a view source link by importing & using `import.meta.url.replace(\"ems.sh\", \"val.town)\"` (or passing this data to the client) and include `target=\"_top\"` attribute\n- **Error Debugging:** Add `` to HTML to capture client-side errors\n- **Error Handling:** Only use try...catch when there's a clear local resolution; Avoid catches that merely log or return 500s. Let errors bubble up with full context\n- **Environment Variables:** Use `Deno.env.get('keyname')` when you need to, but generally prefer APIs that don't require keys\n- **Imports:** Use `https://esm.sh` for npm and Deno dependencies to ensure compatibility on server and browser\n- **Storage Strategy:** Only use backend storage if explicitly required; prefer simple static client-side sites\n- **React Configuration:** When using React libraries, pin versions with `?deps=react@18.2.0,react-dom@18.2.0` and start the file with `/** @jsxImportSource https://esm.sh/react@18.2.0 */`\n- Ensure all React dependencies and sub-dependencies are pinned to the same version\n- **Styling:** Default to using TailwindCSS via `` unless otherwise specified\n\n## Project Structure and Design Patterns\n\n### Recommended Directory Structure\n```\n├── backend/\n│ ├── database/\n│ │ ├── migrations.ts # Schema definitions\n│ │ ├── queries.ts # DB query functions\n│ │ └── README.md\n│ └── routes/ # Route modules\n│ ├── [route].ts\n│ └── static.ts # Static file serving\n│ ├── index.ts # Main entry point\n│ └── README.md\n├── frontend/\n│ ├── components/\n│ │ ├── App.tsx\n│ │ └── [Component].tsx\n│ ├── favicon.svg\n│ ├── index.html # Main HTML template\n│ ├── index.tsx # Frontend JS entry point\n│ ├── README.md\n│ └── style.css\n├── README.md\n└── shared/\n ├── README.md\n └── utils.ts # Shared types and functions\n```\n\n### Backend (Hono) Best Practices\n\n- Hono is the recommended API framework\n- Main entry point should be `backend/index.ts`\n- **Static asset serving:** Use the utility functions to read and serve project files:\n ```ts\n import { readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n \n // serve all files in frontend/ and shared/\n app.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\n app.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n \n // For index.html, often you'll want to bootstrap with initial data\n app.get(\"/\", async c => {\n let html = await readFile(\"/frontend/index.html\", import.meta.url);\n \n // Inject data to avoid extra round-trips\n const initialData = await fetchInitialData();\n const dataScript = ``;\n \n html = html.replace(\"\", `${dataScript}`);\n return c.html(html);\n });\n ```\n- Create RESTful API routes for CRUD operations\n- Always include this snippet at the top-level Hono app to re-throwing errors to see full stack traces:\n ```ts\n // Unwrap Hono errors to see original error details\n app.onError((err, c) => {\n throw err;\n });\n ```\n\n### Database Patterns\n- Run migrations on startup or comment out for performance\n- Change table names when modifying schemas rather than altering\n- Export clear query functions with proper TypeScript typing\n\n## Common Gotchas and Solutions\n\n1. **Environment Limitations:** \n - Val Town runs on Deno in a serverless context, not Node.js\n - Code in `shared/` must work in both frontend and backend environments\n - Cannot use `Deno` keyword in shared code\n - Use `https://esm.sh` for imports that work in both environments\n\n2. **SQLite Peculiarities:**\n - Limited support for ALTER TABLE operations\n - Create new tables with updated schemas and copy data when needed\n - Always run table creation before querying\n\n3. **React Configuration:**\n - All React dependencies must be pinned to 18.2.0\n - Always include `@jsxImportSource https://esm.sh/react@18.2.0` at the top of React files\n - Rendering issues often come from mismatched React versions\n\n4. **File Handling:**\n - Val Town only supports text files, not binary\n - Use the provided utilities to read files across branches and forks\n - For files in the project, use `readFile` helpers\n\n5. **API Design:**\n - `fetch` handler is the entry point for HTTP vals\n - Run the Hono app with `export default app.fetch // This is the entry point for HTTP vals`\n\n",[176],"pulls","branches",2,"forks","envVars",{"_190":143,"_192":62},"connections",{"_190":62,"_191":62},"count","unfilledCount","emptyCount",{"_43":175,"_32":176,"_30":177,"_134":-5,"_41":42,"_40":178,"_135":151,"_38":219,"_13":62,"_33":20,"_35":180,"_24":25,"_138":-5,"_201":220,"_27":221},{"_43":168,"_32":169,"_30":170,"_134":-5,"_41":42,"_40":171,"_135":25,"_38":216,"_13":137,"_33":20,"_35":173,"_24":25,"_138":-5,"_201":217,"_27":218},{"_43":162,"_32":163,"_30":164,"_134":-5,"_41":42,"_40":143,"_135":25,"_38":213,"_13":62,"_33":34,"_35":166,"_24":25,"_138":-5,"_201":214,"_27":215},{"_43":155,"_32":156,"_30":157,"_134":-5,"_41":42,"_40":158,"_135":25,"_38":210,"_13":143,"_33":34,"_35":160,"_24":25,"_138":-5,"_201":211,"_27":212},{"_43":147,"_32":148,"_30":149,"_134":-5,"_41":42,"_40":150,"_135":151,"_38":207,"_13":62,"_33":34,"_35":153,"_24":25,"_138":-5,"_201":208,"_27":209},{"_43":140,"_32":141,"_30":142,"_134":-5,"_41":42,"_40":143,"_135":25,"_38":204,"_13":62,"_33":34,"_35":145,"_24":25,"_138":-5,"_201":205,"_27":206},{"_43":133,"_32":28,"_30":31,"_134":-5,"_41":42,"_40":14,"_135":25,"_38":200,"_13":137,"_33":37,"_35":36,"_24":25,"_138":-5,"_201":202,"_27":203},["D",1760377185915],"children",[],[28],["D",1760276213018],[],[141],["D",1760362505083],[],[148],["D",1760375692732],[],[156],["D",1760276213018],[],[163],["D",1760371403000],[],[169],["D",1760371385620],[],[176],["D",1760276213018],{"_96":95,"_103":99},{"_96":95,"_97":-5,"_98":99,"_18":56,"_100":101,"_102":25},{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"ctx",{"_78":-5,"_123":259,"_260":261,"_262":263,"_264":25,"_265":25,"_266":25,"_267":151,"_268":-5,"_269":-7,"_270":-7,"_271":62,"_272":25,"_273":25,"_274":-5,"_275":25,"_276":25,"_277":25,"_278":-5},"ENV",{"_230":231,"_232":233,"_234":86,"_235":236,"_237":238,"_239":240,"_241":242,"_243":244,"_245":246,"_247":248,"_249":250,"_251":252,"_253":254,"_255":256,"_257":258},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v17