🤖 Reddit Scraper API

API endpoints for the /r/hardwareswap scraper

\n \n

GET /stats

Get scraping statistics and database info

\n \n

POST /scrape

Manually trigger a scraping session

\n \n

GET /health

Check if the scraper is configured correctly

\n \n

Note: The main scraper runs automatically on a cron schedule.

\n \n \n `, {\n headers: { 'Content-Type': 'text/html' }\n });\n\n case '/stats':\n const stats = await scraper.getStats();\n if (!stats) {\n return new Response(JSON.stringify({ error: 'Failed to get stats' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n return new Response(JSON.stringify(stats, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/scrape':\n if (req.method !== 'POST') {\n return new Response(JSON.stringify({ error: 'Method not allowed' }), {\n status: 405,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n \n const results = await scraper.scrapeWithResults();\n return new Response(JSON.stringify(results, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/health':\n // Check if environment variables are set\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n const redditClientId = Deno.env.get('REDDIT_CLIENT_ID');\n const redditClientSecret = Deno.env.get('REDDIT_CLIENT_SECRET');\n \n const health = {\n status: 'ok',\n timestamp: new Date().toISOString(),\n config: {\n supabaseConfigured: !!(supabaseUrl && supabaseKey),\n redditConfigured: !!(redditClientId && redditClientSecret),\n supabaseUrl: supabaseUrl ? '✓ Set' : '✗ Missing',\n supabaseKey: supabaseKey ? '✓ Set' : '✗ Missing',\n redditClientId: redditClientId ? '✓ Set' : '✗ Missing',\n redditClientSecret: redditClientSecret ? '✓ Set' : '✗ Missing'\n }\n };\n\n const allConfigured = health.config.supabaseConfigured && health.config.redditConfigured;\n const status = allConfigured ? 200 : 500;\n if (!allConfigured) {\n health.status = 'error';\n }\n\n return new Response(JSON.stringify(health, null, 2), {\n status,\n headers: { 'Content-Type': 'application/json' }\n });\n\n default:\n return new Response(JSON.stringify({ error: 'Not found' }), {\n status: 404,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n } catch (error) {\n console.error('API Error:', error);\n return new Response(JSON.stringify({ \n error: 'Internal server error',\n message: error.message \n }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n}",[],[191],{"_102":201,"_95":202,"_151":203,"_153":-5,"_154":136,"_155":204,"_157":158,"_104":205,"_99":206,"_161":196,"_163":207,"_165":22,"_166":-5,"_167":208,"_98":209},"e7d2caf2-3a65-11f0-9c78-9e149126039e","test-reddit-oauth.ts","2a43009c-3a65-11f0-b3d8-9e149126039e",31,["D",1748286980380],2,"/**\n * Test script for Reddit OAuth authentication\n * Use this to verify your Reddit credentials work before running the main scraper\n */\n\nexport default async function(req: Request) {\n const clientId = Deno.env.get(\"REDDIT_CLIENT_ID\");\n const clientSecret = Deno.env.get(\"REDDIT_CLIENT_SECRET\");\n const userAgent = Deno.env.get(\"REDDIT_USER_AGENT\") || \"Val Town Reddit Scraper Test 1.0\";\n\n if (!clientId || !clientSecret) {\n return new Response(\n JSON.stringify(\n {\n error: \"Missing credentials\",\n message: \"Please set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET environment variables\",\n setup: {\n clientId: clientId ? \"✓ Set\" : \"✗ Missing\",\n clientSecret: clientSecret ? \"✓ Set\" : \"✗ Missing\",\n },\n },\n null,\n 2,\n ),\n {\n status: 400,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n\n try {\n // Step 1: Get OAuth token\n console.log(\"🔐 Getting OAuth token...\");\n const auth = btoa(`${clientId}:${clientSecret}`);\n\n const tokenResponse = await fetch(\"https://www.reddit.com/api/v1/access_token\", {\n method: \"POST\",\n headers: {\n \"Authorization\": `Basic ${auth}`,\n \"User-Agent\": userAgent,\n \"Content-Type\": \"application/x-www-form-urlencoded\",\n },\n body: \"grant_type=client_credentials\",\n });\n\n if (!tokenResponse.ok) {\n const errorText = await tokenResponse.text();\n return new Response(\n JSON.stringify(\n {\n error: \"OAuth failed\",\n status: tokenResponse.status,\n statusText: tokenResponse.statusText,\n response: errorText,\n },\n null,\n 2,\n ),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n\n const tokenData = await tokenResponse.json();\n console.log(\"✅ OAuth token received\");\n\n // Step 2: Test API call\n console.log(\"📡 Testing API call...\");\n const apiResponse = await fetch(\"https://oauth.reddit.com/r/hardwareswap/new?limit=50\", {\n headers: {\n \"Authorization\": `Bearer ${tokenData.access_token}`,\n \"User-Agent\": userAgent,\n },\n });\n\n if (!apiResponse.ok) {\n const errorText = await apiResponse.text();\n return new Response(\n JSON.stringify(\n {\n error: \"API call failed\",\n status: apiResponse.status,\n statusText: apiResponse.statusText,\n response: errorText,\n },\n null,\n 2,\n ),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n\n const apiData = await apiResponse.json();\n console.log(\"✅ API call successful\");\n\n // Step 3: Parse and return results\n const posts = apiData.data.children.map((child: any) => ({\n id: child.data.id,\n title: child.data.title.substring(0, 100) + (child.data.title.length > 100 ? \"...\" : \"\"),\n author: child.data.author,\n created: new Date(child.data.created_utc * 1000).toISOString(),\n score: child.data.score,\n num_comments: child.data.num_comments,\n }));\n\n return new Response(\n JSON.stringify(\n {\n success: true,\n message: \"Reddit OAuth and API test successful! 🎉\",\n token_info: {\n token_type: tokenData.token_type,\n expires_in: tokenData.expires_in,\n scope: tokenData.scope,\n },\n sample_posts: posts,\n total_posts_available: apiData.data.children.length,\n },\n null,\n 2,\n ),\n {\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n } catch (error) {\n console.error(\"Test failed:\", error);\n return new Response(\n JSON.stringify(\n {\n error: \"Test failed\",\n message: error.message,\n stack: error.stack,\n },\n null,\n 2,\n ),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n}",[],[202],"files",[148,170,179,189,200],"counts",{"_214":32,"_215":216,"_217":32,"_218":219},"pulls","branches",1,"forks","envVars",{"_220":221,"_222":32},"count",4,"emptyCount","contents",[225,228,231,234,237],{"_102":149,"_95":150,"_151":152,"_153":-5,"_154":136,"_155":156,"_157":158,"_104":226,"_99":160,"_161":162,"_163":164,"_165":22,"_166":-5,"_98":227},["D",1748286632385],[150],{"_102":171,"_95":172,"_151":173,"_153":-5,"_154":136,"_155":174,"_157":158,"_104":229,"_99":32,"_161":162,"_163":176,"_165":22,"_166":-5,"_98":230},["D",1748286154154],[172],{"_102":180,"_95":181,"_151":182,"_153":-5,"_154":136,"_155":135,"_157":158,"_104":232,"_99":184,"_161":185,"_163":186,"_165":22,"_166":-5,"_98":233},["D",1748287042314],[181],{"_102":190,"_95":191,"_151":192,"_153":-5,"_154":136,"_155":193,"_157":158,"_104":235,"_99":195,"_161":196,"_163":197,"_165":22,"_166":-5,"_98":236},["D",1748286598470],[191],{"_102":201,"_95":202,"_151":203,"_153":-5,"_154":136,"_155":204,"_157":158,"_104":238,"_99":206,"_161":196,"_163":207,"_165":22,"_166":-5,"_98":239},["D",1748286980380],[202],"upstreamProject","upstreamBranch","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.environment-variables",{"_93":94,"_95":96,"_100":248,"_218":253},{"_102":103,"_95":96,"_104":249,"_106":-5,"_107":22,"_108":109,"_110":-5,"_111":-5,"_14":112,"_113":-5,"_114":22,"_115":-5,"_116":32,"_16":117,"_118":119,"_120":119,"_121":122,"_123":122,"_124":122,"_125":119,"_126":119,"_23":250,"_130":251,"_133":252,"_145":22},["D",1748286043940],{"_93":94,"_128":129},{"_93":94,"_132":129},{"_99":135,"_95":43,"_102":136,"_137":103,"_138":-5,"_139":-5,"_140":-5,"_141":-5,"_142":-5,"_143":96,"_144":94},[254,261,265,269],{"_255":256,"_104":257,"_258":-5,"_110":259,"_260":22},"key","REDDIT_CLIENT_SECRET",["D",1748286730630],"updatedAt","Reddit App Secret https://old.reddit.com/prefs/apps/","isEmpty",{"_255":262,"_104":263,"_258":-5,"_110":264,"_260":22},"REDDIT_CLIENT_ID",["D",1748286706053],"Reddit App ID https://old.reddit.com/prefs/apps/",{"_255":266,"_104":267,"_258":-5,"_110":268,"_260":22},"SUPABASE_ANON_KEY",["D",1748286350956],"API key of our Database",{"_255":270,"_104":271,"_258":-5,"_110":272,"_260":22},"SUPABASE_URL",["D",1748286327411],"URL of our Database","actionData","errors"]

wxw

scrape-hws

Environment variables