[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_189":190,"_191":192},"routes/_app.x.$handle.$name",{"_96":95,"_32":45,"_104":86,"_27":105,"_13":14,"_106":-5,"_11":107,"_108":109,"_110":111,"_112":113,"_114":115,"_20":-5,"_116":-5,"_117":-5,"_118":25,"_119":120,"_121":58,"_122":-5,"_123":-5,"_124":125},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":56,"_57":58,"_59":58,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":25},"version",10,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","reddit-scraper.ts","latestPath","fileId","f63fe63a-3a63-11f0-ac2f-9e149126039e","name","fileType","interval","content","/**\n * Reddit /r/hardwareswap Scraper for Val Town\n * Scrapes posts from Reddit and stores them in Supabase using snoowrap (JavaScript Reddit API wrapper)\n * Set this as a cron job to run periodically\n */\n\nimport { createClient } from \"https://esm.sh/@supabase/supabase-js@2.38.4\";\nimport Snoowrap from \"https://esm.sh/snoowrap@1.23.0\";\n\n// Types\ninterface RedditPost {\n id: string;\n title: string;\n author: string;\n created: number;\n selftext: string;\n url: string;\n score: number;\n num_comments: number;\n permalink: string;\n subreddit: string;\n}\n\ninterface PostRecord {\n reddit_id: string;\n reddit_original: RedditPost;\n title: string;\n created_at: string;\n updated_at: string;\n}\n\nclass RedditScraper {\n private supabase;\n \n constructor() {\n // Initialize Supabase client\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n \n if (!supabaseUrl || !supabaseKey) {\n throw new Error('Missing Supabase credentials. Please set SUPABASE_URL and SUPABASE_ANON_KEY environment variables.');\n }\n \n this.supabase = createClient(supabaseUrl, supabaseKey);\n }\n\n /**\n * Fetch new posts from /r/hardwareswap using Reddit's JSON API\n */\n async fetchNewPosts(): Promise {\n try {\n const response = await fetch('https://www.reddit.com/r/hardwareswap/new.json?limit=25', {\n headers: {\n 'User-Agent': 'Val Town Reddit Scraper 1.0'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Reddit API error: ${response.status} ${response.statusText}`);\n }\n\n const data = await response.json();\n \n return data.data.children.map((child: any) => ({\n id: child.data.id,\n title: child.data.title,\n author: child.data.author,\n created: child.data.created_utc,\n selftext: child.data.selftext,\n url: child.data.url,\n score: child.data.score,\n num_comments: child.data.num_comments,\n permalink: child.data.permalink,\n subreddit: child.data.subreddit\n }));\n } catch (error) {\n console.error('Error fetching posts from Reddit:', error);\n return [];\n }\n }\n\n /**\n * Check if a post already exists in the database\n */\n async postExists(redditId: string): Promise {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('reddit_id')\n .eq('reddit_id', redditId)\n .single();\n\n if (error && error.code !== 'PGRST116') { // PGRST116 is \"not found\"\n console.error('Error checking if post exists:', error);\n return false;\n }\n\n return !!data;\n } catch (error) {\n console.error('Error checking if post exists:', error);\n return false;\n }\n }\n\n /**\n * Save a post to the database\n */\n async savePost(post: RedditPost): Promise {\n try {\n const postRecord: PostRecord = {\n reddit_id: post.id,\n reddit_original: post,\n title: post.title,\n created_at: new Date(post.created * 1000).toISOString(),\n updated_at: new Date().toISOString()\n };\n\n const { error } = await this.supabase\n .from('posts')\n .insert([postRecord]);\n\n if (error) {\n console.error('Error saving post:', error);\n return false;\n }\n\n return true;\n } catch (error) {\n console.error('Error saving post:', error);\n return false;\n }\n }\n\n /**\n * Main scraping function\n */\n async scrape(): Promise {\n const currentTime = new Date().toISOString();\n console.log(`🔍 Scraping /r/hardwareswap... at ${currentTime}`);\n\n const posts = await this.fetchNewPosts();\n let newPostsCount = 0;\n let duplicatePostsCount = 0;\n\n for (const post of posts) {\n try {\n // Check if post already exists to avoid duplicates\n if (await this.postExists(post.id)) {\n duplicatePostsCount++;\n continue;\n }\n\n // Save new post\n const saved = await this.savePost(post);\n if (saved) {\n newPostsCount++;\n console.log(`✅ NEW: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n } else {\n console.log(`❌ FAILED to save: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n }\n } catch (error) {\n console.error(`❌ ERROR processing post ${post.id}:`, error);\n }\n }\n\n // Summary\n const totalPosts = newPostsCount + duplicatePostsCount;\n console.log(`📊 Summary: ${newPostsCount} new posts, ${duplicatePostsCount} duplicates skipped out of ${totalPosts} total posts`);\n }\n}\n\n/**\n * Main function - this will be called by the cron trigger\n */\nexport default async function() {\n try {\n const scraper = new RedditScraper();\n await scraper.scrape();\n } catch (error) {\n console.error('❌ Scraper failed:', error);\n throw error; // Re-throw to ensure cron job shows as failed\n }\n}","/**\n * Reddit /r/hardwareswap Scraper for Val Town\n * Scrapes posts from Reddit and stores them in Supabase using snoowrap (JavaScript Reddit API wrapper)\n * Set this as a cron job to run periodically\n */\n\nimport { createClient } from \"https://esm.sh/@supabase/supabase-js@2.38.4\";\nimport Snoowrap from \"https://esm.sh/snoowrap@1.23.0\";\n\n// Types\ninterface RedditPost {\n id: string;\n title: string;\n author: string;\n created: number;\n selftext: string;\n url: string;\n score: number;\n num_comments: number;\n permalink: string;\n subreddit: string;\n}\n\ninterface PostRecord {\n reddit_id: string;\n reddit_original: RedditPost;\n title: string;\n created_at: string;\n updated_at: string;\n}\n\nclass RedditScraper {\n private supabase;\n private reddit;\n \n constructor() {\n // Initialize Supabase client\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n \n if (!supabaseUrl || !supabaseKey) {\n throw new Error('Missing Supabase credentials. Please set SUPABASE_URL and SUPABASE_ANON_KEY environment variables.');\n }\n \n this.supabase = createClient(supabaseUrl, supabaseKey);\n \n // Initialize Reddit client\n const clientId = Deno.env.get('REDDIT_CLIENT_ID');\n const clientSecret = Deno.env.get('REDDIT_CLIENT_SECRET');\n const userAgent = Deno.env.get('REDDIT_USER_AGENT') || 'Val Town Reddit Scraper 1.0';\n \n if (!clientId || !clientSecret) {\n throw new Error('Missing Reddit credentials. Please set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET environment variables.');\n }\n \n this.reddit = new Snoowrap({\n userAgent: userAgent,\n clientId: clientId,\n clientSecret: clientSecret,\n // Using client credentials flow (no username/password needed)\n grantType: 'client_credentials'\n });\n }\n\n /**\n * Fetch new posts from /r/hardwareswap using Reddit's JSON API\n */\n async fetchNewPosts(): Promise {\n try {\n const response = await fetch('https://www.reddit.com/r/hardwareswap/new.json?limit=25', {\n headers: {\n 'User-Agent': 'Val Town Reddit Scraper 1.0'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Reddit API error: ${response.status} ${response.statusText}`);\n }\n\n const data = await response.json();\n \n return data.data.children.map((child: any) => ({\n id: child.data.id,\n title: child.data.title,\n author: child.data.author,\n created: child.data.created_utc,\n selftext: child.data.selftext,\n url: child.data.url,\n score: child.data.score,\n num_comments: child.data.num_comments,\n permalink: child.data.permalink,\n subreddit: child.data.subreddit\n }));\n } catch (error) {\n console.error('Error fetching posts from Reddit:', error);\n return [];\n }\n }\n\n /**\n * Check if a post already exists in the database\n */\n async postExists(redditId: string): Promise {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('reddit_id')\n .eq('reddit_id', redditId)\n .single();\n\n if (error && error.code !== 'PGRST116') { // PGRST116 is \"not found\"\n console.error('Error checking if post exists:', error);\n return false;\n }\n\n return !!data;\n } catch (error) {\n console.error('Error checking if post exists:', error);\n return false;\n }\n }\n\n /**\n * Save a post to the database\n */\n async savePost(post: RedditPost): Promise {\n try {\n const postRecord: PostRecord = {\n reddit_id: post.id,\n reddit_original: post,\n title: post.title,\n created_at: new Date(post.created * 1000).toISOString(),\n updated_at: new Date().toISOString()\n };\n\n const { error } = await this.supabase\n .from('posts')\n .insert([postRecord]);\n\n if (error) {\n console.error('Error saving post:', error);\n return false;\n }\n\n return true;\n } catch (error) {\n console.error('Error saving post:', error);\n return false;\n }\n }\n\n /**\n * Main scraping function\n */\n async scrape(): Promise {\n const currentTime = new Date().toISOString();\n console.log(`🔍 Scraping /r/hardwareswap... at ${currentTime}`);\n\n const posts = await this.fetchNewPosts();\n let newPostsCount = 0;\n let duplicatePostsCount = 0;\n\n for (const post of posts) {\n try {\n // Check if post already exists to avoid duplicates\n if (await this.postExists(post.id)) {\n duplicatePostsCount++;\n continue;\n }\n\n // Save new post\n const saved = await this.savePost(post);\n if (saved) {\n newPostsCount++;\n console.log(`✅ NEW: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n } else {\n console.log(`❌ FAILED to save: ${post.title.substring(0, 50)}... (ID: ${post.id})`);\n }\n } catch (error) {\n console.error(`❌ ERROR processing post ${post.id}:`, error);\n }\n }\n\n // Summary\n const totalPosts = newPostsCount + duplicatePostsCount;\n console.log(`📊 Summary: ${newPostsCount} new posts, ${duplicatePostsCount} duplicates skipped out of ${totalPosts} total posts`);\n }\n}\n\n/**\n * Main function - this will be called by the cron trigger\n */\nexport default async function() {\n try {\n const scraper = new RedditScraper();\n await scraper.scrape();\n } catch (error) {\n console.error('❌ Scraper failed:', error);\n throw error; // Re-throw to ensure cron job shows as failed\n }\n}","createdAt",["D",1748286447906],"branchVersion","branchId","b9a80662-3a63-11f0-a608-9e149126039e","id","b9a7ae6a-3a63-11f0-a608-9e149126039e","scrape-hws",["D",1748286043940],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","f9d50b19-b05a-4e7f-bad2-53d2e2e27d33","pinnedFileId","2280f040-3a64-11f0-a852-9e149126039e","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","4f6b67ff-c237-43cb-bec0-989e52fb7647","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_96":95,"_103":99},"org",{"_96":95,"_97":-5,"_98":99,"_18":54,"_100":101,"_102":25},"branch",{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"isOwned",32,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","wxw","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJVbmxseXI0N0x5ZFBqeTBhWDg3S0dlb2szZSIsImluaXRpYWxzIjoiV1cifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":185,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":56,"_57":58,"_59":58,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":186,"_80":187,"_82":188,"_84":25},"tree",[168,169,170,171],"files",[168,169,170,171],"counts",{"_158":62,"_159":136,"_160":62,"_161":162,"_163":164},"contents",[126,127,128,129],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":151,"_32":152,"_30":153,"_132":-5,"_41":42,"_40":154,"_134":25,"_38":155,"_13":62,"_33":20,"_35":156,"_24":25,"_139":-5,"_27":157},{"_43":145,"_32":146,"_30":147,"_132":-5,"_41":42,"_40":143,"_134":58,"_38":148,"_13":62,"_33":20,"_35":149,"_24":25,"_139":-5,"_27":150},{"_43":141,"_32":28,"_30":31,"_132":-5,"_41":42,"_40":14,"_134":25,"_38":142,"_13":143,"_33":34,"_35":37,"_24":25,"_139":-5,"_27":144},{"_43":130,"_32":131,"_30":56,"_132":-5,"_41":42,"_40":133,"_134":25,"_38":135,"_13":136,"_33":137,"_35":138,"_24":25,"_139":-5,"_27":140},"275167f8-3a64-11f0-b2e2-9e149126039e","scraper-api.ts","parentId",7,"latest",["D",1748286227911],1,"http","/**\n * HTTP API for Reddit Scraper\n * Provides endpoints to manually trigger scraping and view statistics\n */\n\nimport { createClient } from \"https://esm.sh/@supabase/supabase-js@2.38.4\";\n\n// Import the scraper logic (you might need to refactor reddit-scraper.ts to export the class)\ninterface RedditPost {\n id: string;\n title: string;\n author: string;\n created: number;\n selftext: string;\n url: string;\n score: number;\n num_comments: number;\n permalink: string;\n subreddit: string;\n}\n\ninterface PostRecord {\n reddit_id: string;\n reddit_original: RedditPost;\n title: string;\n created_at: string;\n updated_at: string;\n}\n\nclass RedditScraper {\n private supabase;\n \n constructor() {\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n \n if (!supabaseUrl || !supabaseKey) {\n throw new Error('Missing Supabase credentials');\n }\n \n this.supabase = createClient(supabaseUrl, supabaseKey);\n }\n\n async fetchNewPosts(): Promise {\n try {\n const response = await fetch('https://www.reddit.com/r/hardwareswap/new.json?limit=25', {\n headers: {\n 'User-Agent': 'Val Town Reddit Scraper 1.0'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Reddit API error: ${response.status} ${response.statusText}`);\n }\n\n const data = await response.json();\n \n return data.data.children.map((child: any) => ({\n id: child.data.id,\n title: child.data.title,\n author: child.data.author,\n created: child.data.created_utc,\n selftext: child.data.selftext,\n url: child.data.url,\n score: child.data.score,\n num_comments: child.data.num_comments,\n permalink: child.data.permalink,\n subreddit: child.data.subreddit\n }));\n } catch (error) {\n console.error('Error fetching posts from Reddit:', error);\n return [];\n }\n }\n\n async postExists(redditId: string): Promise {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('reddit_id')\n .eq('reddit_id', redditId)\n .single();\n\n if (error && error.code !== 'PGRST116') {\n console.error('Error checking if post exists:', error);\n return false;\n }\n\n return !!data;\n } catch (error) {\n console.error('Error checking if post exists:', error);\n return false;\n }\n }\n\n async savePost(post: RedditPost): Promise {\n try {\n const postRecord: PostRecord = {\n reddit_id: post.id,\n reddit_original: post,\n title: post.title,\n created_at: new Date(post.created * 1000).toISOString(),\n updated_at: new Date().toISOString()\n };\n\n const { error } = await this.supabase\n .from('posts')\n .insert([postRecord]);\n\n if (error) {\n console.error('Error saving post:', error);\n return false;\n }\n\n return true;\n } catch (error) {\n console.error('Error saving post:', error);\n return false;\n }\n }\n\n async getStats() {\n try {\n const { data, error } = await this.supabase\n .from('posts')\n .select('id, created_at, reddit_original')\n .order('created_at', { ascending: false });\n\n if (error) {\n throw error;\n }\n\n const totalPosts = data.length;\n const today = new Date();\n today.setHours(0, 0, 0, 0);\n \n const postsToday = data.filter(post => \n new Date(post.created_at) >= today\n ).length;\n\n const lastWeek = new Date();\n lastWeek.setDate(lastWeek.getDate() - 7);\n \n const postsThisWeek = data.filter(post => \n new Date(post.created_at) >= lastWeek\n ).length;\n\n return {\n totalPosts,\n postsToday,\n postsThisWeek,\n latestPost: data[0] ? {\n title: data[0].reddit_original.title,\n created_at: data[0].created_at,\n author: data[0].reddit_original.author\n } : null\n };\n } catch (error) {\n console.error('Error getting stats:', error);\n return null;\n }\n }\n\n async scrapeWithResults() {\n const startTime = Date.now();\n const posts = await this.fetchNewPosts();\n let newPostsCount = 0;\n let duplicatePostsCount = 0;\n const errors: string[] = [];\n\n for (const post of posts) {\n try {\n if (await this.postExists(post.id)) {\n duplicatePostsCount++;\n continue;\n }\n\n const saved = await this.savePost(post);\n if (saved) {\n newPostsCount++;\n } else {\n errors.push(`Failed to save post ${post.id}`);\n }\n } catch (error) {\n errors.push(`Error processing post ${post.id}: ${error.message}`);\n }\n }\n\n const endTime = Date.now();\n const duration = endTime - startTime;\n\n return {\n success: true,\n duration,\n totalPosts: posts.length,\n newPostsCount,\n duplicatePostsCount,\n errors,\n timestamp: new Date().toISOString()\n };\n }\n}\n\nexport default async function(req: Request) {\n const url = new URL(req.url);\n const path = url.pathname;\n\n try {\n const scraper = new RedditScraper();\n\n // Handle different endpoints\n switch (path) {\n case '/':\n return new Response(`\n \n \n Reddit Scraper API\n \n \n \n

🤖 Reddit Scraper API

API endpoints for the /r/hardwareswap scraper

\n \n

GET /stats

Get scraping statistics and database info

\n \n

POST /scrape

Manually trigger a scraping session

\n \n

GET /health

Check if the scraper is configured correctly

\n \n

Note: The main scraper runs automatically on a cron schedule.

\n \n \n `, {\n headers: { 'Content-Type': 'text/html' }\n });\n\n case '/stats':\n const stats = await scraper.getStats();\n if (!stats) {\n return new Response(JSON.stringify({ error: 'Failed to get stats' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n return new Response(JSON.stringify(stats, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/scrape':\n if (req.method !== 'POST') {\n return new Response(JSON.stringify({ error: 'Method not allowed' }), {\n status: 405,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n \n const results = await scraper.scrapeWithResults();\n return new Response(JSON.stringify(results, null, 2), {\n headers: { 'Content-Type': 'application/json' }\n });\n\n case '/health':\n // Check if environment variables are set\n const supabaseUrl = Deno.env.get('SUPABASE_URL');\n const supabaseKey = Deno.env.get('SUPABASE_ANON_KEY');\n \n const health = {\n status: 'ok',\n timestamp: new Date().toISOString(),\n config: {\n supabaseConfigured: !!(supabaseUrl && supabaseKey),\n supabaseUrl: supabaseUrl ? '✓ Set' : '✗ Missing',\n supabaseKey: supabaseKey ? '✓ Set' : '✗ Missing'\n }\n };\n\n const status = health.config.supabaseConfigured ? 200 : 500;\n if (!health.config.supabaseConfigured) {\n health.status = 'error';\n }\n\n return new Response(JSON.stringify(health, null, 2), {\n status,\n headers: { 'Content-Type': 'application/json' }\n });\n\n default:\n return new Response(JSON.stringify({ error: 'Not found' }), {\n status: 404,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n } catch (error) {\n console.error('API Error:', error);\n return new Response(JSON.stringify({ \n error: 'Internal server error',\n message: error.message \n }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' }\n });\n }\n}","label",[131],"aa70a126-3a64-11f0-9e55-9e149126039e",["D",1748286447906],3,[28],"fb590502-3a63-11f0-865b-9e149126039e","database-schema.sql","fb5905b6-3a63-11f0-865b-9e149126039e",["D",1748286154154],"-- Supabase Database Schema for Reddit Scraper\n-- Run this SQL in your Supabase SQL editor to create the posts table\n\nCREATE TABLE IF NOT EXISTS posts (\n id BIGSERIAL PRIMARY KEY,\n reddit_id VARCHAR(20) UNIQUE NOT NULL,\n reddit_original JSONB NOT NULL,\n title TEXT NOT NULL,\n created_at TIMESTAMP WITH TIME ZONE NOT NULL,\n updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()\n);\n\n-- Create indexes for better performance\nCREATE INDEX IF NOT EXISTS idx_posts_reddit_id ON posts(reddit_id);\nCREATE INDEX IF NOT EXISTS idx_posts_created_at ON posts(created_at DESC);\nCREATE INDEX IF NOT EXISTS idx_posts_title ON posts USING gin(to_tsvector('english', title));\n\n-- Add a trigger to automatically update the updated_at column\nCREATE OR REPLACE FUNCTION update_updated_at_column()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.updated_at = NOW();\n RETURN NEW;\nEND;\n$$ language 'plpgsql';\n\nCREATE TRIGGER update_posts_updated_at \n BEFORE UPDATE ON posts \n FOR EACH ROW \n EXECUTE FUNCTION update_updated_at_column();\n\n-- Optional: Add RLS (Row Level Security) policies if needed\n-- ALTER TABLE posts ENABLE ROW LEVEL SECURITY;\n\n-- Example policy (uncomment if you want to restrict access)\n-- CREATE POLICY \"Allow all operations for authenticated users\" ON posts\n-- FOR ALL USING (auth.role() = 'authenticated');",[146],"07a997fe-3a64-11f0-b88f-9e149126039e","README.md","07a99966-3a64-11f0-b88f-9e149126039e",4,["D",1748286174814],"# Reddit /r/hardwareswap Scraper\n\nA Val Town application that scrapes posts from Reddit's /r/hardwareswap subreddit and stores them in a Supabase database.\n\n## Features\n\n- 🔄 Automated scraping of /r/hardwareswap posts\n- 🗄️ Stores posts in Supabase PostgreSQL database\n- 🚫 Duplicate detection to avoid storing the same post twice\n- 📊 Detailed logging and statistics\n- ⏰ Runs on a cron schedule (configurable in Val Town UI)\n\n## Setup Instructions\n\n### 1. Supabase Setup\n\n1. Create a new project in [Supabase](https://supabase.com)\n2. Go to the SQL Editor in your Supabase dashboard\n3. Copy and paste the contents of `database-schema.sql` and run it\n4. Go to Settings > API to get your project URL and anon key\n\n### 2. Environment Variables\n\nSet these environment variables in your Val Town settings:\n\n- `SUPABASE_URL`: Your Supabase project URL (e.g., `https://your-project.supabase.co`)\n- `SUPABASE_ANON_KEY`: Your Supabase anon/public key\n\n### 3. Configure Cron Schedule\n\n1. Set `reddit-scraper.ts` as a cron trigger in Val Town\n2. Configure the schedule in the Val Town web UI (recommended: every 30 minutes)\n3. Example cron expressions:\n - Every 30 minutes: `*/30 * * * *`\n - Every hour: `0 * * * *`\n - Every 15 minutes: `*/15 * * * *`\n\n## Database Schema\n\nThe `posts` table contains:\n\n- `id`: Primary key (auto-increment)\n- `reddit_id`: Unique Reddit post ID\n- `reddit_original`: Full Reddit post data as JSON\n- `title`: Post title\n- `created_at`: When the post was created on Reddit\n- `updated_at`: When the record was last updated in our database\n\n## Usage\n\n### Manual Run\n\nYou can manually trigger the scraper by running the `reddit-scraper.ts` val.\n\n### Automated Run\n\nOnce configured as a cron job, it will automatically:\n\n1. Fetch the latest 25 posts from /r/hardwareswap\n2. Check for duplicates in the database\n3. Save new posts to Supabase\n4. Log statistics about the scraping session\n\n## API Alternative\n\nIf you prefer to use Reddit's official API instead of the JSON endpoint:\n\n1. Create a Reddit app at https://www.reddit.com/prefs/apps\n2. Add these environment variables:\n - `REDDIT_CLIENT_ID`\n - `REDDIT_CLIENT_SECRET`\n - `REDDIT_USER_AGENT`\n3. Modify the scraper to use the official Reddit API\n\n## Monitoring\n\nCheck the Val Town logs to monitor:\n- Number of new posts scraped\n- Number of duplicates skipped\n- Any errors during scraping\n- Performance metrics\n\n## Troubleshooting\n\n### Common Issues\n\n1. **Missing environment variables**: Ensure `SUPABASE_URL` and `SUPABASE_ANON_KEY` are set\n2. **Database connection errors**: Verify your Supabase credentials and that the table exists\n3. **Reddit rate limiting**: The scraper uses a 25-post limit and respectful user agent\n4. **Duplicate key errors**: The scraper checks for duplicates, but race conditions might occur\n\n### Error Messages\n\n- `Missing Supabase credentials`: Set the required environment variables\n- `Reddit API error`: Check if Reddit is accessible and not rate limiting\n- `Error saving post`: Check Supabase connection and table schema\n\n## Data Access\n\nYou can query your scraped data directly in Supabase:\n\n```sql\n-- Get recent posts\nSELECT title, created_at, reddit_original->>'score' as score \nFROM posts \nORDER BY created_at DESC \nLIMIT 10;\n\n-- Search posts by title\nSELECT title, created_at \nFROM posts \nWHERE title ILIKE '%gpu%' \nORDER BY created_at DESC;\n\n-- Get posts by author\nSELECT title, created_at \nFROM posts \nWHERE reddit_original->>'author' = 'username' \nORDER BY created_at DESC;\n```\n\n## Contributing\n\nFeel free to modify the scraper to:\n- Add more subreddits\n- Include additional post metadata\n- Add data processing or analysis features\n- Integrate with other services",[152],"pulls","branches","forks","envVars",{"_165":154,"_167":62},"connections",{"_165":62,"_166":62},"count","unfilledCount","emptyCount",{"_43":151,"_32":152,"_30":153,"_132":-5,"_41":42,"_40":154,"_134":25,"_38":182,"_13":62,"_33":20,"_35":156,"_24":25,"_139":-5,"_173":183,"_27":184},{"_43":145,"_32":146,"_30":147,"_132":-5,"_41":42,"_40":143,"_134":58,"_38":179,"_13":62,"_33":20,"_35":149,"_24":25,"_139":-5,"_173":180,"_27":181},{"_43":141,"_32":28,"_30":31,"_132":-5,"_41":42,"_40":14,"_134":25,"_38":176,"_13":143,"_33":34,"_35":37,"_24":25,"_139":-5,"_173":177,"_27":178},{"_43":130,"_32":131,"_30":56,"_132":-5,"_41":42,"_40":133,"_134":25,"_38":172,"_13":136,"_33":137,"_35":138,"_24":25,"_139":-5,"_173":174,"_27":175},["D",1748286227911],"children",[],[131],["D",1748286447906],[],[28],["D",1748286154154],[],[146],["D",1748286174814],[],[152],["D",1748286043940],{"_96":95,"_103":99},{"_96":95,"_97":-5,"_98":99,"_18":54,"_100":101,"_102":25},{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"ctx",{"_78":-5,"_123":222,"_223":224,"_225":226,"_227":25,"_228":25,"_229":25,"_230":58,"_231":-5,"_232":-7,"_233":-7,"_234":62,"_235":25,"_236":25,"_237":-5,"_238":25,"_239":25,"_240":25,"_241":-5},"ENV",{"_193":194,"_195":196,"_197":86,"_198":199,"_200":201,"_202":203,"_204":205,"_206":207,"_208":209,"_210":211,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e8c4b7030cb7600e4eb0736b15f422c8b8bd130f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v10