🤖 Reddit Scraper API

API endpoints for the /r/hardwareswap scraper

\n \n

GET /stats

Get scraping statistics and database info

\n \n

POST /scrape

Manually trigger a scraping session

\n \n

GET /health

Check if the scraper is configured correctly

\n \n

Note: The main scraper runs automatically on a cron schedule.

\n \n \n `, {\n headers: { 'Content-Type': 'text/html' }\n });\n\n case '/stats':\n const stats = await scraper.getStats();\n if (!stats) {\n return new Response(JSON.stringify({ error: 'Failed to get stats' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n return new Response(JSON.stringify(stats, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/scrape':\n if (req.method !== 'POST') {\n return new Response(JSON.stringify({ error: 'Method not allowed' }), {\n status: 405,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n \n const results = await scraper.scrapeWithResults();\n return new Response(JSON.stringify(results, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/health':\n // Check if environment variables are set\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n const redditClientId = Deno.env.get('REDDIT_CLIENT_ID');\n const redditClientSecret = Deno.env.get('REDDIT_CLIENT_SECRET');\n \n const health = {\n status: 'ok',\n timestamp: new Date().toISOString(),\n config: {\n supabaseConfigured: !!(supabaseUrl && supabaseKey),\n redditConfigured: !!(redditClientId && redditClientSecret),\n supabaseUrl: supabaseUrl ? '✓ Set' : '✗ Missing',\n supabaseKey: supabaseKey ? '✓ Set' : '✗ Missing',\n redditClientId: redditClientId ? '✓ Set' : '✗ Missing',\n redditClientSecret: redditClientSecret ? '✓ Set' : '✗ Missing'\n }\n };\n\n const allConfigured = health.config.supabaseConfigured && health.config.redditConfigured;\n const status = allConfigured ? 200 : 500;\n if (!allConfigured) {\n health.status = 'error';\n }\n\n return new Response(JSON.stringify(health, null, 2), {\n status,\n headers: { 'Content-Type': 'application/json' }\n });\n\n default:\n return new Response(JSON.stringify({ error: 'Not found' }), {\n status: 404,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n } catch (error) {\n console.error('API Error:', error);\n return new Response(JSON.stringify({ \n error: 'Internal server error',\n message: error.message \n }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n}",[136],"0cbc7426-3a66-11f0-8dd5-9e149126039e","reddit-scraper.ts","f63fe63a-3a63-11f0-ac2f-9e149126039e",["D",1748287042314],9,"interval","/**\n * Reddit /r/hardwareswap Scraper for Val Town\n * Scrapes posts from Reddit and stores them in Supabase using Reddit's OAuth API\n * Set this as a cron job to run periodically\n */\n\nimport { createClient } from \"https://esm.sh/@supabase/supabase-js@2.38.4\";\n\n// Types\ninterface RedditPost {\n id: string;\n title: string;\n author: string;\n created: number;\n selftext: string;\n url: string;\n score: number;\n num_comments: number;\n permalink: string;\n subreddit: string;\n}\n\ninterface PostRecord {\n reddit_id: string;\n reddit_original: RedditPost;\n title: string;\n created_at: string;\n updated_at: string;\n}\n\nclass RedditScraper {\n private supabase;\n private clientId: string;\n private clientSecret: string;\n private userAgent: string;\n private accessToken: string | null = null;\n\n constructor() {\n // Initialize Supabase client\n const supabaseUrl = Deno.env.get(\"SUPABASE_URL\");\n const supabaseKey = Deno.env.get(\"SUPABASE_ANON_KEY\");\n\n if (!supabaseUrl || !supabaseKey) {\n throw new Error(\n \"Missing Supabase credentials. Please set SUPABASE_URL and SUPABASE_ANON_KEY environment variables.\",\n );\n }\n\n this.supabase = createClient(supabaseUrl, supabaseKey);\n\n // Initialize Reddit credentials\n this.clientId = Deno.env.get(\"REDDIT_CLIENT_ID\") || \"\";\n this.clientSecret = Deno.env.get(\"REDDIT_CLIENT_SECRET\") || \"\";\n this.userAgent = Deno.env.get(\"REDDIT_USER_AGENT\") || \"Val Town Reddit Scraper 1.0\";\n\n if (!this.clientId || !this.clientSecret) {\n throw new Error(\n \"Missing Reddit credentials. Please set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET environment variables.\",\n );\n }\n }\n\n /**\n * Get OAuth access token using client credentials flow\n */\n async getAccessToken(): Promise {\n if (this.accessToken) {\n return this.accessToken;\n }\n\n try {\n const auth = btoa(`${this.clientId}:${this.clientSecret}`);\n\n const response = await fetch(\"https://www.reddit.com/api/v1/access_token\", {\n method: \"POST\",\n headers: {\n \"Authorization\": `Basic ${auth}`,\n \"User-Agent\": this.userAgent,\n \"Content-Type\": \"application/x-www-form-urlencoded\",\n },\n body: \"grant_type=client_credentials\",\n });\n\n if (!response.ok) {\n throw new Error(`OAuth error: ${response.status} ${response.statusText}`);\n }\n\n const data = await response.json();\n this.accessToken = data.access_token;\n\n if (!this.accessToken) {\n throw new Error(\"No access token received from Reddit\");\n }\n\n return this.accessToken;\n } catch (error) {\n console.error(\"Error getting Reddit access token:\", error);\n throw error;\n }\n }\n\n /**\n * Fetch new posts from /r/hardwareswap using Reddit's OAuth API\n */\n async fetchNewPosts(): Promise {\n try {\n const accessToken = await this.getAccessToken();\n\n const response = await fetch(\"https://oauth.reddit.com/r/hardwareswap/new?limit=100\", {\n headers: {\n \"Authorization\": `Bearer ${accessToken}`,\n \"User-Agent\": this.userAgent,\n },\n });\n\n if (!response.ok) {\n // If token expired, clear it and retry once\n if (response.status === 401) {\n this.accessToken = null;\n const newToken = await this.getAccessToken();\n\n const retryResponse = await fetch(\"https://oauth.reddit.com/r/hardwareswap/new?limit=100\", {\n headers: {\n \"Authorization\": `Bearer ${newToken}`,\n \"User-Agent\": this.userAgent,\n },\n });\n\n if (!retryResponse.ok) {\n throw new Error(`Reddit API error: ${retryResponse.status} ${retryResponse.statusText}`);\n }\n\n const retryData = await retryResponse.json();\n return this.parseRedditResponse(retryData);\n }\n\n throw new Error(`Reddit API error: ${response.status} ${response.statusText}`);\n }\n\n const data = await response.json();\n return this.parseRedditResponse(data);\n } catch (error) {\n console.error(\"Error fetching posts from Reddit:\", error);\n return [];\n }\n }\n\n /**\n * Parse Reddit API response into our RedditPost format\n */\n private parseRedditResponse(data: any): RedditPost[] {\n if (!data.data || !data.data.children) {\n console.error(\"Unexpected Reddit API response format\");\n return [];\n }\n\n return data.data.children.map((child: any) => ({\n id: child.data.id,\n title: child.data.title,\n author: child.data.author || \"[deleted]\",\n created: child.data.created_utc,\n selftext: child.data.selftext || \"\",\n url: child.data.url,\n score: child.data.score,\n num_comments: child.data.num_comments,\n permalink: child.data.permalink,\n subreddit: child.data.subreddit,\n }));\n }\n\n /**\n * Check if a post already exists in the database\n */\n async postExists(redditId: string): Promise {\n try {\n const { data, error } = await this.supabase\n .from(\"posts\")\n .select(\"reddit_id\")\n .eq(\"reddit_id\", redditId)\n .single();\n\n if (error && error.code !== \"PGRST116\") { // PGRST116 is \"not found\"\n console.error(\"Error checking if post exists:\", error);\n return false;\n }\n\n return !!data;\n } catch (error) {\n console.error(\"Error checking if post exists:\", error);\n return false;\n }\n }\n\n /**\n * Save a post to the database\n */\n async savePost(post: RedditPost): Promise {\n try {\n const postRecord: PostRecord = {\n reddit_id: post.id,\n reddit_original: post,\n title: post.title,\n created_at: new Date(post.created * 1000).toISOString(),\n updated_at: new Date().toISOString(),\n };\n\n const { error } = await this.supabase\n .from(\"posts\")\n .insert([postRecord]);\n\n if (error) {\n console.error(\"Error saving post:\", error);\n return false;\n }\n\n return true;\n } catch (error) {\n console.error(\"Error saving post:\", error);\n return false;\n }\n }\n\n /**\n * Main scraping function\n */\n async scrape(): Promise {\n const currentTime = new Date().toISOString();\n console.log(`🔍 Scraping /r/hardwareswap... at ${currentTime}`);\n\n const posts = await this.fetchNewPosts();\n let newPostsCount = 0;\n let duplicatePostsCount = 0;\n\n for (const post of posts) {\n try {\n // Check if post already exists to avoid duplicates\n if (await this.postExists(post.id)) {\n duplicatePostsCount++;\n continue;\n }\n\n // Save new post\n const saved = await this.savePost(post);\n if (saved) {\n newPostsCount++;\n console.log(`✅ NEW: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n } else {\n console.log(`❌ FAILED to save: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n }\n } catch (error) {\n console.error(`❌ ERROR processing post ${post.id}:`, error);\n }\n }\n\n // Summary\n const totalPosts = newPostsCount + duplicatePostsCount;\n console.log(\n `📊 Summary: ${newPostsCount} new posts, ${duplicatePostsCount} duplicates skipped out of ${totalPosts} total posts`,\n );\n }\n}\n\n/**\n * Main function - this will be called by the cron trigger\n */\nexport default async function() {\n try {\n const scraper = new RedditScraper();\n await scraper.scrape();\n } catch (error) {\n console.error(\"❌ Scraper failed:\", error);\n throw error; // Re-throw to ensure cron job shows as failed\n }\n}",[143],"fb590502-3a63-11f0-865b-9e149126039e","database-schema.sql","fb5905b6-3a63-11f0-865b-9e149126039e",3,["D",1748286154154],"-- Supabase Database Schema for Reddit Scraper\n-- Run this SQL in your Supabase SQL editor to create the posts table\n\nCREATE TABLE IF NOT EXISTS posts (\n id BIGSERIAL PRIMARY KEY,\n reddit_id VARCHAR(20) UNIQUE NOT NULL,\n reddit_original JSONB NOT NULL,\n title TEXT NOT NULL,\n created_at TIMESTAMP WITH TIME ZONE NOT NULL,\n updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()\n);\n\n-- Create indexes for better performance\nCREATE INDEX IF NOT EXISTS idx_posts_reddit_id ON posts(reddit_id);\nCREATE INDEX IF NOT EXISTS idx_posts_created_at ON posts(created_at DESC);\nCREATE INDEX IF NOT EXISTS idx_posts_title ON posts USING gin(to_tsvector('english', title));\n\n-- Add a trigger to automatically update the updated_at column\nCREATE OR REPLACE FUNCTION update_updated_at_column()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.updated_at = NOW();\n RETURN NEW;\nEND;\n$$ language 'plpgsql';\n\nCREATE TRIGGER update_posts_updated_at \n BEFORE UPDATE ON posts \n FOR EACH ROW \n EXECUTE FUNCTION update_updated_at_column();\n\n-- Optional: Add RLS (Row Level Security) policies if needed\n-- ALTER TABLE posts ENABLE ROW LEVEL SECURITY;\n\n-- Example policy (uncomment if you want to restrict access)\n-- CREATE POLICY \"Allow all operations for authenticated users\" ON posts\n-- FOR ALL USING (auth.role() = 'authenticated');",[151],"18659ca4-3a65-11f0-8116-9e149126039e","README.md","07a99966-3a64-11f0-b88f-9e149126039e",27,["D",1748286632385],6,"# Reddit /r/hardwareswap Scraper\n\nA Val Town application that scrapes posts from Reddit's /r/hardwareswap subreddit and stores them in a Supabase database.\n\n## Features\n\n- 🔄 Automated scraping of /r/hardwareswap posts using Reddit's official OAuth API\n- 🗄️ Stores posts in Supabase PostgreSQL database\n- 🚫 Duplicate detection to avoid storing the same post twice\n- 📊 Detailed logging and statistics\n- ⏰ Runs on a cron schedule (configurable in Val Town UI)\n- 🔐 Secure OAuth authentication with automatic token refresh\n\n## Setup Instructions\n\n### 1. Reddit API Setup\n\n1. Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps)\n2. Click \"Create App\" or \"Create Another App\"\n3. Fill out the form:\n - **Name**: Your app name (e.g., \"Val Town Scraper\")\n - **App type**: Select \"script\"\n - **Description**: Optional description\n - **About URL**: Leave blank or add your website\n - **Redirect URI**: Use `http://localhost:8080` (required but not used)\n4. Click \"Create app\"\n5. Note down your **Client ID** (under the app name) and **Client Secret**\n\n### 2. Supabase Setup\n\n1. Create a new project in [Supabase](https://supabase.com)\n2. Go to the SQL Editor in your Supabase dashboard\n3. Copy and paste the contents of `database-schema.sql` and run it\n4. Go to Settings > API to get your project URL and anon key\n\n### 3. Environment Variables\n\nSet these environment variables in your Val Town settings:\n\n**Supabase:**\n- `SUPABASE_URL`: Your Supabase project URL (e.g., `https://your-project.supabase.co`)\n- `SUPABASE_ANON_KEY`: Your Supabase anon/public key\n\n**Reddit API:**\n- `REDDIT_CLIENT_ID`: Your Reddit app's client ID\n- `REDDIT_CLIENT_SECRET`: Your Reddit app's client secret\n- `REDDIT_USER_AGENT`: Optional custom user agent (defaults to \"Val Town Reddit Scraper 1.0\")\n\n### 4. Configure Cron Schedule\n\n1. Set `reddit-scraper.ts` as a cron trigger in Val Town\n2. Configure the schedule in the Val Town web UI (recommended: every 30 minutes)\n3. Example cron expressions:\n - Every 30 minutes: `*/30 * * * *`\n - Every hour: `0 * * * *`\n - Every 15 minutes: `*/15 * * * *`\n\n## Database Schema\n\nThe `posts` table contains:\n\n- `id`: Primary key (auto-increment)\n- `reddit_id`: Unique Reddit post ID\n- `reddit_original`: Full Reddit post data as JSON\n- `title`: Post title\n- `created_at`: When the post was created on Reddit\n- `updated_at`: When the record was last updated in our database\n\n## Usage\n\n### Manual Run\n\nYou can manually trigger the scraper by running the `reddit-scraper.ts` val.\n\n### Automated Run\n\nOnce configured as a cron job, it will automatically:\n\n1. Authenticate with Reddit using OAuth client credentials\n2. Fetch the latest 25 posts from /r/hardwareswap\n3. Check for duplicates in the database\n4. Save new posts to Supabase\n5. Log statistics about the scraping session\n\n## Technical Details\n\n### OAuth Flow\n\nThe scraper uses Reddit's **Client Credentials** OAuth flow:\n\n1. Authenticates using your app's client ID and secret\n2. Receives an access token from Reddit\n3. Uses the token to make authenticated API requests\n4. Automatically refreshes the token if it expires\n\nThis approach is more reliable than using Reddit's public JSON endpoints and respects Reddit's rate limits.\n\n### Rate Limiting\n\n- Reddit allows 60 requests per minute for OAuth applications\n- The scraper fetches 25 posts per run, well within limits\n- Recommended cron schedule: every 30 minutes or longer\n\n## Monitoring\n\nCheck the Val Town logs to monitor:\n- Number of new posts scraped\n- Number of duplicates skipped\n- Any errors during scraping\n- Performance metrics\n\n## Troubleshooting\n\n### Common Issues\n\n1. **Missing environment variables**: Ensure all required Reddit and Supabase credentials are set\n2. **Database connection errors**: Verify your Supabase credentials and that the table exists\n3. **Reddit OAuth errors**: Check your Reddit app credentials and ensure the app type is \"script\"\n4. **Rate limiting**: Reddit may temporarily block requests if rate limits are exceeded\n5. **Duplicate key errors**: The scraper checks for duplicates, but race conditions might occur\n\n### Error Messages\n\n- `Missing Supabase credentials`: Set SUPABASE_URL and SUPABASE_ANON_KEY\n- `Missing Reddit credentials`: Set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET\n- `OAuth error`: Check your Reddit app credentials and app type\n- `Reddit API error`: May indicate rate limiting or API issues\n- `Error saving post`: Check Supabase connection and table schema\n\n## Data Access\n\nYou can query your scraped data directly in Supabase:\n\n```sql\n-- Get recent posts\nSELECT title, created_at, reddit_original->>'score' as score \nFROM posts \nORDER BY created_at DESC \nLIMIT 10;\n\n-- Search posts by title\nSELECT title, created_at \nFROM posts \nWHERE title ILIKE '%gpu%' \nORDER BY created_at DESC;\n\n-- Get posts by author\nSELECT title, created_at \nFROM posts \nWHERE reddit_original->>'author' = 'username' \nORDER BY created_at DESC;\n```\n\n## Contributing\n\nFeel free to modify the scraper to:\n- Add more subreddits\n- Include additional post metadata\n- Add data processing or analysis features\n- Integrate with other services",[158],"pulls",1,"forks","envVars",{"_172":174,"_175":53},"connections",{"_172":53,"_173":53},"count","unfilledCount",4,"emptyCount",{"_20":157,"_13":158,"_119":159,"_121":-5,"_122":21,"_123":160,"_125":49,"_22":194,"_29":162,"_128":101,"_130":163,"_132":38,"_133":-5,"_182":195,"_89":196},{"_20":150,"_13":151,"_119":152,"_121":-5,"_122":21,"_123":153,"_125":49,"_22":191,"_29":53,"_128":101,"_130":155,"_132":38,"_133":-5,"_182":192,"_89":193},{"_20":142,"_13":143,"_119":144,"_121":-5,"_122":21,"_123":30,"_125":49,"_22":188,"_29":146,"_128":147,"_130":148,"_132":38,"_133":-5,"_182":189,"_89":190},{"_20":135,"_13":136,"_119":47,"_121":-5,"_122":21,"_123":137,"_125":49,"_22":185,"_29":139,"_128":129,"_130":140,"_132":38,"_133":-5,"_182":186,"_89":187},{"_20":117,"_13":118,"_119":120,"_121":-5,"_122":21,"_123":124,"_125":49,"_22":181,"_29":127,"_128":129,"_130":131,"_132":38,"_133":-5,"_182":183,"_89":184},["D",1748286980380],"children",[],[118],["D",1748286598470],[],[136],["D",1748287042314],[],[143],["D",1748286154154],[],[151],["D",1748286632385],[],[158],["D",1748286043940],{"_11":12,"_87":83},{"_11":12,"_81":-5,"_82":83,"_44":45,"_84":85,"_86":38},{"_29":30,"_13":26,"_20":21,"_27":28,"_31":-5,"_32":-5,"_76":-5,"_77":-5,"_78":-5,"_79":14,"_80":12},"ctx",{"_69":-5,"_109":234,"_235":236,"_237":238,"_239":38,"_240":38,"_241":38,"_242":49,"_243":-5,"_244":-7,"_245":-7,"_246":53,"_247":38,"_248":38,"_249":-5,"_250":38,"_251":38,"_252":38,"_253":-5},"ENV",{"_205":206,"_207":208,"_209":26,"_210":211,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221,"_222":223,"_224":225,"_226":227,"_228":229,"_230":231,"_232":233},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e8c4b7030cb7600e4eb0736b15f422c8b8bd130f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Branches1

Branches
1