[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_191":192,"_193":194},"routes/_app.x.$handle.$name",{"_95":94,"_32":44,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":57,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":25,"_48":49,"_50":49,"_51":-5,"_52":-5,"_18":53,"_54":55,"_56":57,"_58":57,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",16,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":20,"_34":36,"_37":38,"_39":14,"_40":41,"_24":25},"previous",{"_30":31,"_32":28,"_33":20,"_34":35},"deleted",false,"deletedHere","path","README.md","latestPath","fileId","07a99966-3a64-11f0-b88f-9e149126039e","name","fileType","content","# Reddit /r/hardwareswap Scraper\n\nA Val Town application that scrapes posts from Reddit's /r/hardwareswap subreddit and stores them in a Supabase database.\n\n## Features\n\n- 🔄 Automated scraping of /r/hardwareswap posts\n- 🗄️ Stores posts in Supabase PostgreSQL database\n- 🚫 Duplicate detection to avoid storing the same post twice\n- 📊 Detailed logging and statistics\n- ⏰ Runs on a cron schedule (configurable in Val Town UI)\n\n## Setup Instructions\n\n### 1. Supabase Setup\n\n1. Create a new project in [Supabase](https://supabase.com)\n2. Go to the SQL Editor in your Supabase dashboard\n3. Copy and paste the contents of `database-schema.sql` and run it\n4. Go to Settings > API to get your project URL and anon key\n\n### 2. Environment Variables\n\nSet these environment variables in your Val Town settings:\n\n- `SUPABASE_URL`: Your Supabase project URL (e.g., `https://your-project.supabase.co`)\n- `SUPABASE_ANON_KEY`: Your Supabase anon/public key\n\n### 3. Configure Cron Schedule\n\n1. Set `reddit-scraper.ts` as a cron trigger in Val Town\n2. Configure the schedule in the Val Town web UI (recommended: every 30 minutes)\n3. Example cron expressions:\n - Every 30 minutes: `*/30 * * * *`\n - Every hour: `0 * * * *`\n - Every 15 minutes: `*/15 * * * *`\n\n## Database Schema\n\nThe `posts` table contains:\n\n- `id`: Primary key (auto-increment)\n- `reddit_id`: Unique Reddit post ID\n- `reddit_original`: Full Reddit post data as JSON\n- `title`: Post title\n- `created_at`: When the post was created on Reddit\n- `updated_at`: When the record was last updated in our database\n\n## Usage\n\n### Manual Run\n\nYou can manually trigger the scraper by running the `reddit-scraper.ts` val.\n\n### Automated Run\n\nOnce configured as a cron job, it will automatically:\n\n1. Fetch the latest 25 posts from /r/hardwareswap\n2. Check for duplicates in the database\n3. Save new posts to Supabase\n4. Log statistics about the scraping session\n\n## API Alternative\n\nIf you prefer to use Reddit's official API instead of the JSON endpoint:\n\n1. Create a Reddit app at https://www.reddit.com/prefs/apps\n2. Add these environment variables:\n - `REDDIT_CLIENT_ID`\n - `REDDIT_CLIENT_SECRET`\n - `REDDIT_USER_AGENT`\n3. Modify the scraper to use the official Reddit API\n\n## Monitoring\n\nCheck the Val Town logs to monitor:\n- Number of new posts scraped\n- Number of duplicates skipped\n- Any errors during scraping\n- Performance metrics\n\n## Troubleshooting\n\n### Common Issues\n\n1. **Missing environment variables**: Ensure `SUPABASE_URL` and `SUPABASE_ANON_KEY` are set\n2. **Database connection errors**: Verify your Supabase credentials and that the table exists\n3. **Reddit rate limiting**: The scraper uses a 25-post limit and respectful user agent\n4. **Duplicate key errors**: The scraper checks for duplicates, but race conditions might occur\n\n### Error Messages\n\n- `Missing Supabase credentials`: Set the required environment variables\n- `Reddit API error`: Check if Reddit is accessible and not rate limiting\n- `Error saving post`: Check Supabase connection and table schema\n\n## Data Access\n\nYou can query your scraped data directly in Supabase:\n\n```sql\n-- Get recent posts\nSELECT title, created_at, reddit_original->>'score' as score \nFROM posts \nORDER BY created_at DESC \nLIMIT 10;\n\n-- Search posts by title\nSELECT title, created_at \nFROM posts \nWHERE title ILIKE '%gpu%' \nORDER BY created_at DESC;\n\n-- Get posts by author\nSELECT title, created_at \nFROM posts \nWHERE reddit_original->>'author' = 'username' \nORDER BY created_at DESC;\n```\n\n## Contributing\n\nFeel free to modify the scraper to:\n- Add more subreddits\n- Include additional post metadata\n- Add data processing or analysis features\n- Integrate with other services","# Reddit /r/hardwareswap Scraper\n\nA Val Town application that scrapes posts from Reddit's /r/hardwareswap subreddit and stores them in a Supabase database.\n\n## Features\n\n- 🔄 Automated scraping of /r/hardwareswap posts\n- 🗄️ Stores posts in Supabase PostgreSQL database\n- 🚫 Duplicate detection to avoid storing the same post twice\n- 📊 Detailed logging and statistics\n- ⏰ Runs on a cron schedule (configurable in Val Town UI)\n\n## Setup Instructions\n\n### 1. Reddit API Setup\n\n1. Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps)\n2. Click \"Create App\" or \"Create Another App\"\n3. Fill out the form:\n - **Name**: Your app name (e.g., \"Val Town Scraper\")\n - **App type**: Select \"script\"\n - **Description**: Optional description\n - **About URL**: Leave blank or add your website\n - **Redirect URI**: Use `http://localhost:8080` (required but not used)\n4. Click \"Create app\"\n5. Note down your **Client ID** (under the app name) and **Client Secret**\n\n### 2. Supabase Setup\n\n1. Create a new project in [Supabase](https://supabase.com)\n2. Go to the SQL Editor in your Supabase dashboard\n3. Copy and paste the contents of `database-schema.sql` and run it\n4. Go to Settings > API to get your project URL and anon key\n\n### 3. Environment Variables\n\nSet these environment variables in your Val Town settings:\n\n**Supabase:**\n- `SUPABASE_URL`: Your Supabase project URL (e.g., `https://your-project.supabase.co`)\n- `SUPABASE_ANON_KEY`: Your Supabase anon/public key\n\n**Reddit API:**\n- `REDDIT_CLIENT_ID`: Your Reddit app's client ID\n- `REDDIT_CLIENT_SECRET`: Your Reddit app's client secret\n- `REDDIT_USER_AGENT`: Optional custom user agent (defaults to \"Val Town Reddit Scraper 1.0\")\n\n1. Set `reddit-scraper.ts` as a cron trigger in Val Town\n2. Configure the schedule in the Val Town web UI (recommended: every 30 minutes)\n3. Example cron expressions:\n - Every 30 minutes: `*/30 * * * *`\n - Every hour: `0 * * * *`\n - Every 15 minutes: `*/15 * * * *`\n\n## Database Schema\n\nThe `posts` table contains:\n\n- `id`: Primary key (auto-increment)\n- `reddit_id`: Unique Reddit post ID\n- `reddit_original`: Full Reddit post data as JSON\n- `title`: Post title\n- `created_at`: When the post was created on Reddit\n- `updated_at`: When the record was last updated in our database\n\n## Usage\n\n### Manual Run\n\nYou can manually trigger the scraper by running the `reddit-scraper.ts` val.\n\n### Automated Run\n\nOnce configured as a cron job, it will automatically:\n\n1. Fetch the latest 25 posts from /r/hardwareswap\n2. Check for duplicates in the database\n3. Save new posts to Supabase\n4. Log statistics about the scraping session\n\n## API Alternative\n\nIf you prefer to use Reddit's official API instead of the JSON endpoint:\n\n1. Create a Reddit app at https://www.reddit.com/prefs/apps\n2. Add these environment variables:\n - `REDDIT_CLIENT_ID`\n - `REDDIT_CLIENT_SECRET`\n - `REDDIT_USER_AGENT`\n3. Modify the scraper to use the official Reddit API\n\n## Monitoring\n\nCheck the Val Town logs to monitor:\n- Number of new posts scraped\n- Number of duplicates skipped\n- Any errors during scraping\n- Performance metrics\n\n## Troubleshooting\n\n### Common Issues\n\n1. **Missing environment variables**: Ensure `SUPABASE_URL` and `SUPABASE_ANON_KEY` are set\n2. **Database connection errors**: Verify your Supabase credentials and that the table exists\n3. **Reddit rate limiting**: The scraper uses a 25-post limit and respectful user agent\n4. **Duplicate key errors**: The scraper checks for duplicates, but race conditions might occur\n\n### Error Messages\n\n- `Missing Supabase credentials`: Set the required environment variables\n- `Reddit API error`: Check if Reddit is accessible and not rate limiting\n- `Error saving post`: Check Supabase connection and table schema\n\n## Data Access\n\nYou can query your scraped data directly in Supabase:\n\n```sql\n-- Get recent posts\nSELECT title, created_at, reddit_original->>'score' as score \nFROM posts \nORDER BY created_at DESC \nLIMIT 10;\n\n-- Search posts by title\nSELECT title, created_at \nFROM posts \nWHERE title ILIKE '%gpu%' \nORDER BY created_at DESC;\n\n-- Get posts by author\nSELECT title, created_at \nFROM posts \nWHERE reddit_original->>'author' = 'username' \nORDER BY created_at DESC;\n```\n\n## Contributing\n\nFeel free to modify the scraper to:\n- Add more subreddits\n- Include additional post metadata\n- Add data processing or analysis features\n- Integrate with other services","createdAt",["D",1748286506551],"branchVersion","branchId","b9a80662-3a63-11f0-a608-9e149126039e","id","b9a7ae6a-3a63-11f0-a608-9e149126039e","scrape-hws",["D",1748286043940],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","f9d50b19-b05a-4e7f-bad2-53d2e2e27d33","pinnedFileId","2280f040-3a64-11f0-a852-9e149126039e","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","4f6b67ff-c237-43cb-bec0-989e52fb7647","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":53,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"isOwned",32,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","wxw","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJVbmxseXI0N0x5ZFBqeTBhWDg3S0dlb2szZSIsImluaXRpYWxzIjoiV1cifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_42":43,"_32":44,"_37":187,"_46":-5,"_47":25,"_48":49,"_50":49,"_51":-5,"_52":-5,"_18":53,"_54":55,"_56":57,"_58":57,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":188,"_79":189,"_81":190,"_83":25},"tree",[170,171,172,173],"files",[170,171,172,173],"counts",{"_160":61,"_161":158,"_162":61,"_163":164,"_165":166},"contents",[125,126,127,128],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":156,"_32":28,"_30":31,"_131":-5,"_40":41,"_39":14,"_133":25,"_37":157,"_13":158,"_33":20,"_34":36,"_24":25,"_138":-5,"_27":159},{"_42":149,"_32":150,"_30":151,"_131":-5,"_40":41,"_39":152,"_133":57,"_37":153,"_13":61,"_33":20,"_34":154,"_24":25,"_138":-5,"_27":155},{"_42":140,"_32":141,"_30":142,"_131":-5,"_40":41,"_39":143,"_133":25,"_37":144,"_13":145,"_33":146,"_34":147,"_24":25,"_138":-5,"_27":148},{"_42":129,"_32":130,"_30":55,"_131":-5,"_40":41,"_39":132,"_133":25,"_37":134,"_13":135,"_33":136,"_34":137,"_24":25,"_138":-5,"_27":139},"c55f0b94-3a64-11f0-ac2f-9e149126039e","scraper-api.ts","parentId",15,"latest",["D",1748286493090],5,"http","/**\n * HTTP API for Reddit Scraper\n * Provides endpoints to manually trigger scraping and view statistics\n */\n\nimport { createClient } from \"https://esm.sh/@supabase/supabase-js@2.38.4\";\nimport Snoowrap from \"https://esm.sh/snoowrap@1.23.0\";\n\n// Import the scraper logic (you might need to refactor reddit-scraper.ts to export the class)\ninterface RedditPost {\n id: string;\n title: string;\n author: string;\n created: number;\n selftext: string;\n url: string;\n score: number;\n num_comments: number;\n permalink: string;\n subreddit: string;\n}\n\ninterface PostRecord {\n reddit_id: string;\n reddit_original: RedditPost;\n title: string;\n created_at: string;\n updated_at: string;\n}\n\nclass RedditScraper {\n private supabase;\n private reddit;\n \n constructor() {\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n \n if (!supabaseUrl || !supabaseKey) {\n throw new Error('Missing Supabase credentials');\n }\n \n this.supabase = createClient(supabaseUrl, supabaseKey);\n \n // Initialize Reddit client\n const clientId = Deno.env.get('REDDIT_CLIENT_ID');\n const clientSecret = Deno.env.get('REDDIT_CLIENT_SECRET');\n const userAgent = Deno.env.get('REDDIT_USER_AGENT') || 'Val Town Reddit Scraper 1.0';\n \n if (!clientId || !clientSecret) {\n throw new Error('Missing Reddit credentials');\n }\n \n this.reddit = new Snoowrap({\n userAgent: userAgent,\n clientId: clientId,\n clientSecret: clientSecret,\n grantType: 'client_credentials'\n });\n }\n\n async fetchNewPosts(): Promise {\n try {\n const subreddit = this.reddit.getSubreddit('hardwareswap');\n const submissions = await subreddit.getNew({ limit: 25 });\n \n return submissions.map((submission: any) => ({\n id: submission.id,\n title: submission.title,\n author: submission.author ? submission.author.name : '[deleted]',\n created: submission.created_utc,\n selftext: submission.selftext || '',\n url: submission.url,\n score: submission.score,\n num_comments: submission.num_comments,\n permalink: submission.permalink,\n subreddit: submission.subreddit.display_name\n }));\n } catch (error) {\n console.error('Error fetching posts from Reddit:', error);\n return [];\n }\n }\n\n async postExists(redditId: string): Promise {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('reddit_id')\n .eq('reddit_id', redditId)\n .single();\n\n if (error && error.code !== 'PGRST116') {\n console.error('Error checking if post exists:', error);\n return false;\n }\n\n return !!data;\n } catch (error) {\n console.error('Error checking if post exists:', error);\n return false;\n }\n }\n\n async savePost(post: RedditPost): Promise {\n try {\n const postRecord: PostRecord = {\n reddit_id: post.id,\n reddit_original: post,\n title: post.title,\n created_at: new Date(post.created * 1000).toISOString(),\n updated_at: new Date().toISOString()\n };\n\n const { error } = await this.supabase\n .from('posts')\n .insert([postRecord]);\n\n if (error) {\n console.error('Error saving post:', error);\n return false;\n }\n\n return true;\n } catch (error) {\n console.error('Error saving post:', error);\n return false;\n }\n }\n\n async getStats() {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('id, created_at, reddit_original')\n .order('created_at', { ascending: false });\n\n if (error) {\n throw error;\n }\n\n const totalPosts = data.length;\n const today = new Date();\n today.setHours(0, 0, 0, 0);\n \n const postsToday = data.filter(post => \n new Date(post.created_at) >= today\n ).length;\n\n const lastWeek = new Date();\n lastWeek.setDate(lastWeek.getDate() - 7);\n \n const postsThisWeek = data.filter(post => \n new Date(post.created_at) >= lastWeek\n ).length;\n\n return {\n totalPosts,\n postsToday,\n postsThisWeek,\n latestPost: data[0] ? {\n title: data[0].reddit_original.title,\n created_at: data[0].created_at,\n author: data[0].reddit_original.author\n } : null\n };\n } catch (error) {\n console.error('Error getting stats:', error);\n return null;\n }\n }\n\n async scrapeWithResults() {\n const startTime = Date.now();\n const posts = await this.fetchNewPosts();\n let newPostsCount = 0;\n let duplicatePostsCount = 0;\n const errors: string[] = [];\n\n for (const post of posts) {\n try {\n if (await this.postExists(post.id)) {\n duplicatePostsCount++;\n continue;\n }\n\n const saved = await this.savePost(post);\n if (saved) {\n newPostsCount++;\n } else {\n errors.push(`Failed to save post ${post.id}`);\n }\n } catch (error) {\n errors.push(`Error processing post ${post.id}: ${error.message}`);\n }\n }\n\n const endTime = Date.now();\n const duration = endTime - startTime;\n\n return {\n success: true,\n duration,\n totalPosts: posts.length,\n newPostsCount,\n duplicatePostsCount,\n errors,\n timestamp: new Date().toISOString()\n };\n }\n}\n\nexport default async function(req: Request) {\n const url = new URL(req.url);\n const path = url.pathname;\n\n try {\n const scraper = new RedditScraper();\n\n // Handle different endpoints\n switch (path) {\n case '/':\n return new Response(`\n \n \n Reddit Scraper API\n \n \n \n

🤖 Reddit Scraper API

API endpoints for the /r/hardwareswap scraper

\n \n

GET /stats

Get scraping statistics and database info

\n \n

POST /scrape

Manually trigger a scraping session

\n \n

GET /health

Check if the scraper is configured correctly

\n \n

Note: The main scraper runs automatically on a cron schedule.

\n \n \n `, {\n headers: { 'Content-Type': 'text/html' }\n });\n\n case '/stats':\n const stats = await scraper.getStats();\n if (!stats) {\n return new Response(JSON.stringify({ error: 'Failed to get stats' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n return new Response(JSON.stringify(stats, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/scrape':\n if (req.method !== 'POST') {\n return new Response(JSON.stringify({ error: 'Method not allowed' }), {\n status: 405,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n \n const results = await scraper.scrapeWithResults();\n return new Response(JSON.stringify(results, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/health':\n // Check if environment variables are set\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n const redditClientId = Deno.env.get('REDDIT_CLIENT_ID');\n const redditClientSecret = Deno.env.get('REDDIT_CLIENT_SECRET');\n \n const health = {\n status: 'ok',\n timestamp: new Date().toISOString(),\n config: {\n supabaseConfigured: !!(supabaseUrl && supabaseKey),\n redditConfigured: !!(redditClientId && redditClientSecret),\n supabaseUrl: supabaseUrl ? '✓ Set' : '✗ Missing',\n supabaseKey: supabaseKey ? '✓ Set' : '✗ Missing',\n redditClientId: redditClientId ? '✓ Set' : '✗ Missing',\n redditClientSecret: redditClientSecret ? '✓ Set' : '✗ Missing'\n }\n };\n\n const allConfigured = health.config.supabaseConfigured && health.config.redditConfigured;\n const status = allConfigured ? 200 : 500;\n if (!allConfigured) {\n health.status = 'error';\n }\n\n return new Response(JSON.stringify(health, null, 2), {\n status,\n headers: { 'Content-Type': 'application/json' }\n });\n\n default:\n return new Response(JSON.stringify({ error: 'Not found' }), {\n status: 404,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n } catch (error) {\n console.error('API Error:', error);\n return new Response(JSON.stringify({ \n error: 'Internal server error',\n message: error.message \n }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n}","label",[130],"b05cf12a-3a64-11f0-afdf-9e149126039e","reddit-scraper.ts","f63fe63a-3a63-11f0-ac2f-9e149126039e",11,["D",1748286457842],4,"interval","/**\n * Reddit /r/hardwareswap Scraper for Val Town\n * Scrapes posts from Reddit and stores them in Supabase using snoowrap (JavaScript Reddit API wrapper)\n * Set this as a cron job to run periodically\n */\n\nimport { createClient } from \"https://esm.sh/@supabase/supabase-js@2.38.4\";\nimport Snoowrap from \"https://esm.sh/snoowrap@1.23.0\";\n\n// Types\ninterface RedditPost {\n id: string;\n title: string;\n author: string;\n created: number;\n selftext: string;\n url: string;\n score: number;\n num_comments: number;\n permalink: string;\n subreddit: string;\n}\n\ninterface PostRecord {\n reddit_id: string;\n reddit_original: RedditPost;\n title: string;\n created_at: string;\n updated_at: string;\n}\n\nclass RedditScraper {\n private supabase;\n private reddit;\n \n constructor() {\n // Initialize Supabase client\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n \n if (!supabaseUrl || !supabaseKey) {\n throw new Error('Missing Supabase credentials. Please set SUPABASE_URL and SUPABASE_ANON_KEY environment variables.');\n }\n \n this.supabase = createClient(supabaseUrl, supabaseKey);\n \n // Initialize Reddit client\n const clientId = Deno.env.get('REDDIT_CLIENT_ID');\n const clientSecret = Deno.env.get('REDDIT_CLIENT_SECRET');\n const userAgent = Deno.env.get('REDDIT_USER_AGENT') || 'Val Town Reddit Scraper 1.0';\n \n if (!clientId || !clientSecret) {\n throw new Error('Missing Reddit credentials. Please set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET environment variables.');\n }\n \n this.reddit = new Snoowrap({\n userAgent: userAgent,\n clientId: clientId,\n clientSecret: clientSecret,\n // Using client credentials flow (no username/password needed)\n grantType: 'client_credentials'\n });\n }\n\n /**\n * Fetch new posts from /r/hardwareswap using snoowrap\n */\n async fetchNewPosts(): Promise {\n try {\n const subreddit = this.reddit.getSubreddit('hardwareswap');\n const submissions = await subreddit.getNew({ limit: 25 });\n \n return submissions.map((submission: any) => ({\n id: submission.id,\n title: submission.title,\n author: submission.author ? submission.author.name : '[deleted]',\n created: submission.created_utc,\n selftext: submission.selftext || '',\n url: submission.url,\n score: submission.score,\n num_comments: submission.num_comments,\n permalink: submission.permalink,\n subreddit: submission.subreddit.display_name\n }));\n } catch (error) {\n console.error('Error fetching posts from Reddit:', error);\n return [];\n }\n }\n\n /**\n * Check if a post already exists in the database\n */\n async postExists(redditId: string): Promise {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('reddit_id')\n .eq('reddit_id', redditId)\n .single();\n\n if (error && error.code !== 'PGRST116') { // PGRST116 is \"not found\"\n console.error('Error checking if post exists:', error);\n return false;\n }\n\n return !!data;\n } catch (error) {\n console.error('Error checking if post exists:', error);\n return false;\n }\n }\n\n /**\n * Save a post to the database\n */\n async savePost(post: RedditPost): Promise {\n try {\n const postRecord: PostRecord = {\n reddit_id: post.id,\n reddit_original: post,\n title: post.title,\n created_at: new Date(post.created * 1000).toISOString(),\n updated_at: new Date().toISOString()\n };\n\n const { error } = await this.supabase\n .from('posts')\n .insert([postRecord]);\n\n if (error) {\n console.error('Error saving post:', error);\n return false;\n }\n\n return true;\n } catch (error) {\n console.error('Error saving post:', error);\n return false;\n }\n }\n\n /**\n * Main scraping function\n */\n async scrape(): Promise {\n const currentTime = new Date().toISOString();\n console.log(`🔍 Scraping /r/hardwareswap... at ${currentTime}`);\n\n const posts = await this.fetchNewPosts();\n let newPostsCount = 0;\n let duplicatePostsCount = 0;\n\n for (const post of posts) {\n try {\n // Check if post already exists to avoid duplicates\n if (await this.postExists(post.id)) {\n duplicatePostsCount++;\n continue;\n }\n\n // Save new post\n const saved = await this.savePost(post);\n if (saved) {\n newPostsCount++;\n console.log(`✅ NEW: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n } else {\n console.log(`❌ FAILED to save: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n }\n } catch (error) {\n console.error(`❌ ERROR processing post ${post.id}:`, error);\n }\n }\n\n // Summary\n const totalPosts = newPostsCount + duplicatePostsCount;\n console.log(`📊 Summary: ${newPostsCount} new posts, ${duplicatePostsCount} duplicates skipped out of ${totalPosts} total posts`);\n }\n}\n\n/**\n * Main function - this will be called by the cron trigger\n */\nexport default async function() {\n try {\n const scraper = new RedditScraper();\n await scraper.scrape();\n } catch (error) {\n console.error('❌ Scraper failed:', error);\n throw error; // Re-throw to ensure cron job shows as failed\n }\n}",[141],"fb590502-3a63-11f0-865b-9e149126039e","database-schema.sql","fb5905b6-3a63-11f0-865b-9e149126039e",3,["D",1748286154154],"-- Supabase Database Schema for Reddit Scraper\n-- Run this SQL in your Supabase SQL editor to create the posts table\n\nCREATE TABLE IF NOT EXISTS posts (\n id BIGSERIAL PRIMARY KEY,\n reddit_id VARCHAR(20) UNIQUE NOT NULL,\n reddit_original JSONB NOT NULL,\n title TEXT NOT NULL,\n created_at TIMESTAMP WITH TIME ZONE NOT NULL,\n updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()\n);\n\n-- Create indexes for better performance\nCREATE INDEX IF NOT EXISTS idx_posts_reddit_id ON posts(reddit_id);\nCREATE INDEX IF NOT EXISTS idx_posts_created_at ON posts(created_at DESC);\nCREATE INDEX IF NOT EXISTS idx_posts_title ON posts USING gin(to_tsvector('english', title));\n\n-- Add a trigger to automatically update the updated_at column\nCREATE OR REPLACE FUNCTION update_updated_at_column()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.updated_at = NOW();\n RETURN NEW;\nEND;\n$$ language 'plpgsql';\n\nCREATE TRIGGER update_posts_updated_at \n BEFORE UPDATE ON posts \n FOR EACH ROW \n EXECUTE FUNCTION update_updated_at_column();\n\n-- Optional: Add RLS (Row Level Security) policies if needed\n-- ALTER TABLE posts ENABLE ROW LEVEL SECURITY;\n\n-- Example policy (uncomment if you want to restrict access)\n-- CREATE POLICY \"Allow all operations for authenticated users\" ON posts\n-- FOR ALL USING (auth.role() = 'authenticated');",[150],"cd650a0a-3a64-11f0-a4a2-9e149126039e",["D",1748286506551],1,[28],"pulls","branches","forks","envVars",{"_167":145,"_169":61},"connections",{"_167":61,"_168":61},"count","unfilledCount","emptyCount",{"_42":156,"_32":28,"_30":31,"_131":-5,"_40":41,"_39":14,"_133":25,"_37":184,"_13":158,"_33":20,"_34":36,"_24":25,"_138":-5,"_175":185,"_27":186},{"_42":149,"_32":150,"_30":151,"_131":-5,"_40":41,"_39":152,"_133":57,"_37":181,"_13":61,"_33":20,"_34":154,"_24":25,"_138":-5,"_175":182,"_27":183},{"_42":140,"_32":141,"_30":142,"_131":-5,"_40":41,"_39":143,"_133":25,"_37":178,"_13":145,"_33":146,"_34":147,"_24":25,"_138":-5,"_175":179,"_27":180},{"_42":129,"_32":130,"_30":55,"_131":-5,"_40":41,"_39":132,"_133":25,"_37":174,"_13":135,"_33":136,"_34":137,"_24":25,"_138":-5,"_175":176,"_27":177},["D",1748286493090],"children",[],[130],["D",1748286457842],[],[141],["D",1748286154154],[],[150],["D",1748286506551],[],[28],["D",1748286043940],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":53,"_99":100,"_101":25},{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"ctx",{"_77":-5,"_122":224,"_225":226,"_227":228,"_229":25,"_230":25,"_231":25,"_232":57,"_233":-5,"_234":-7,"_235":-7,"_236":61,"_237":25,"_238":25,"_239":-5,"_240":25,"_241":25,"_242":25,"_243":-5},"ENV",{"_195":196,"_197":198,"_199":85,"_200":201,"_202":203,"_204":205,"_206":207,"_208":209,"_210":211,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221,"_222":223},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e8c4b7030cb7600e4eb0736b15f422c8b8bd130f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v16