[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_260":261,"_262":263},"routes/_app.x.$handle.$name",{"_97":96,"_32":46,"_105":87,"_27":106,"_13":14,"_107":-5,"_11":108,"_109":110,"_111":112,"_113":114,"_115":116,"_20":-5,"_117":-5,"_118":-5,"_119":25,"_120":121,"_122":59,"_123":-5,"_124":-5,"_125":126},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":80,"_81":82,"_83":84,"_85":25},"version",53,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","backend/index.ts","latestPath","fileId","219af112-7ace-11f0-a140-0224a6c84d84","name","index.ts","fileType","script","content","import { Hono } from \"https://esm.sh/hono@3.11.7\";\nimport { stringify } from \"https://esm.sh/csv-stringify@6.4.6/sync\";\nimport { scrapeAllAuctionItems } from \"./scrapers/mainScraper.ts\";\n\nconst app = new Hono();\n\n// Unwrap Hono errors to see original error details\napp.onError((err) => Promise.reject(err));\n\n// Main endpoint - returns all items from all auctions\napp.get(\"/\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n return c.json(data, {\n headers: {\n \"Cache-Control\": \"public, max-age=300\", // Reduced cache for testing\n }\n });\n});\n\n// Health check endpoint\napp.get(\"/health\", (c) => {\n return c.json({ \n status: \"ok\", \n timestamp: new Date().toISOString(),\n service: \"Quarter Price Auction Scraper\"\n });\n});\n\n// API info endpoint\napp.get(\"/info\", (c) => {\n return c.json({\n name: \"Quarter Price Auction Scraper API\",\n description: \"Simple scraper for all items across all auctions from quarterpriceauction.com\",\n endpoints: {\n \"/\": \"Get ALL items from ALL auctions (JSON)\",\n \"/csv\": \"Get ALL items from ALL auctions (CSV format)\",\n \"/health\": \"Health check\",\n \"/info\": \"API information\",\n \"/fresh\": \"Force fresh scrape (no cache)\",\n \"/metadata\": \"Get metadata and sample items only\"\n },\n version: \"3.0.0\",\n features: [\n \"Scrapes every item from every auction\",\n \"Handles pagination automatically\",\n \"Extracts detailed item information\",\n \"JSON and CSV output formats\",\n \"High-performance concurrent processing\"\n ]\n });\n});\n\n// CSV endpoint - returns all items as CSV\napp.get(\"/csv\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n if (data.error) {\n return c.json({ error: data.error }, 500);\n }\n \n // Convert items to CSV format\n const csvData = stringify(data.items, {\n header: true,\n columns: [\n { key: 'title', header: 'Title' },\n { key: 'auctionTitle', header: 'Auction' },\n { key: 'lotNumber', header: 'Lot Number' },\n { key: 'currentBid', header: 'Current Bid' },\n { key: 'startingBid', header: 'Starting Bid' },\n { key: 'bidCount', header: 'Bid Count' },\n { key: 'timeLeft', header: 'Time Left' },\n { key: 'category', header: 'Category' },\n { key: 'auctionUrl', header: 'Auction URL' },\n { key: 'itemUrl', header: 'Item URL' },\n { key: 'imageUrl', header: 'Image URL' },\n { key: 'description', header: 'Description' }\n ]\n });\n \n return c.text(csvData, {\n headers: {\n \"Content-Type\": \"text/csv\",\n \"Content-Disposition\": `attachment; filename=\"quarter-price-auction-items-${new Date().toISOString().split('T')[0]}.csv\"`,\n \"Cache-Control\": \"public, max-age=300\",\n }\n });\n});\n\n// Fresh scrape endpoint (no cache)\napp.get(\"/fresh\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n return c.json(data, {\n headers: {\n \"Cache-Control\": \"no-cache, no-store, must-revalidate\",\n }\n });\n});\n\n// Metadata only endpoint for performance testing\napp.get(\"/metadata\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n return c.json({\n timestamp: data.timestamp,\n totalItems: data.totalItems,\n totalAuctions: data.totalAuctions,\n metadata: data.metadata,\n sampleItems: data.items.slice(0, 3) // Just show first 3 items\n }, {\n headers: {\n \"Cache-Control\": \"no-cache, no-store, must-revalidate\",\n }\n });\n});\n\nexport default app.fetch;","import { Hono } from \"https://esm.sh/hono@3.11.7\";\nimport { stringify } from \"https://esm.sh/csv-stringify@6.4.6/sync\";\nimport { scrapeAllAuctionItems } from \"./scrapers/mainScraper.ts\";\n\nconst app = new Hono();\n\n// Unwrap Hono errors to see original error details\napp.onError((err) => Promise.reject(err));\n\n// Main endpoint - returns all items from all auctions\napp.get(\"/\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n return c.json(data, {\n headers: {\n \"Cache-Control\": \"public, max-age=300\", // Reduced cache for testing\n }\n });\n});\n\n// Health check endpoint\napp.get(\"/health\", (c) => {\n return c.json({ \n status: \"ok\", \n timestamp: new Date().toISOString(),\n service: \"Quarter Price Auction Scraper\"\n });\n});\n\n// API info endpoint\napp.get(\"/info\", (c) => {\n return c.json({\n name: \"Quarter Price Auction Scraper API\",\n description: \"Simple scraper for all items across all auctions from quarterpriceauction.com\",\n endpoints: {\n \"/\": \"Get ALL items from ALL auctions (JSON)\",\n \"/csv\": \"Get ALL items from ALL auctions (CSV format)\",\n \"/health\": \"Health check\",\n \"/info\": \"API information\",\n \"/fresh\": \"Force fresh scrape (no cache)\",\n \"/metadata\": \"Get metadata and sample items only\"\n },\n version: \"3.0.0\",\n features: [\n \"Scrapes every item from every auction\",\n \"Handles pagination automatically\",\n \"Extracts detailed item information\",\n \"JSON and CSV output formats\",\n \"High-performance concurrent processing\"\n ]\n });\n});\n\n// CSV endpoint - returns all items as CSV\napp.get(\"/csv\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n if (data.error) {\n return c.json({ error: data.error }, 500);\n }\n \n // Convert items to CSV format\n const csvData = stringify(data.items, {\n header: true,\n columns: [\n { key: 'title', header: 'Title' },\n { key: 'auctionTitle', header: 'Auction' },\n { key: 'lotNumber', header: 'Lot Number' },\n { key: 'currentBid', header: 'Current Bid' },\n { key: 'startingBid', header: 'Starting Bid' },\n { key: 'bidCount', header: 'Bid Count' },\n { key: 'timeLeft', header: 'Time Left' },\n { key: 'category', header: 'Category' },\n { key: 'auctionUrl', header: 'Auction URL' },\n { key: 'itemUrl', header: 'Item URL' },\n { key: 'imageUrl', header: 'Image URL' },\n { key: 'description', header: 'Description' }\n ]\n });\n \n return c.text(csvData, {\n headers: {\n \"Content-Type\": \"text/csv; charset=utf-8\",\n \"Content-Disposition\": `attachment; filename=\"quarter-price-auction-items-${new Date().toISOString().split('T')[0]}.csv\"`,\n \"Cache-Control\": \"public, max-age=300\",\n }\n });\n});\n\n// Fresh scrape endpoint (no cache)\napp.get(\"/fresh\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n return c.json(data, {\n headers: {\n \"Cache-Control\": \"no-cache, no-store, must-revalidate\",\n }\n });\n});\n\n// Metadata only endpoint for performance testing\napp.get(\"/metadata\", async (c) => {\n const data = await scrapeAllAuctionItems();\n \n return c.json({\n timestamp: data.timestamp,\n totalItems: data.totalItems,\n totalAuctions: data.totalAuctions,\n metadata: data.metadata,\n sampleItems: data.items.slice(0, 3) // Just show first 3 items\n }, {\n headers: {\n \"Cache-Control\": \"no-cache, no-store, must-revalidate\",\n }\n });\n});\n\nexport default app.fetch;","createdAt",["D",1755369531261],"branchVersion","branchId","27c8a162-7612-11f0-80a7-0224a6c84d84","id","27c86a44-7612-11f0-80a7-0224a6c84d84","QPAScrape",["D",1754848079739],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","4555dd0e-707e-4444-acd3-049d206eb1db","pinnedFileId","471c270a-7612-11f0-bbfb-0224a6c84d84","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","bced381e-9ab2-4b55-929d-e7c5064f7f38","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_97":96,"_104":100},"org",{"_97":96,"_98":-5,"_99":100,"_18":55,"_101":102,"_103":25},"branch",{"_13":86,"_32":87,"_44":43,"_88":45,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":46,"_95":96},"isOwned",79,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","wolf","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJvVXpPVUZRTjZGTGd2Q2t4c0RRdFpNZWJ5UCJ9","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":256,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":257,"_81":258,"_83":259,"_85":25},"tree",[173,181,183,184],"files",[173,174,175,176,177,178,179,180,181,182,183,184],"counts",{"_162":63,"_163":164,"_165":63,"_166":167,"_168":169},"contents",[127,128,129,130],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":156,"_32":157,"_30":158,"_132":-5,"_42":43,"_41":159,"_134":59,"_39":160,"_13":63,"_34":154,"_36":-5,"_24":25,"_139":-5,"_27":161},{"_44":149,"_32":150,"_30":151,"_132":-5,"_42":43,"_41":152,"_134":59,"_39":153,"_13":63,"_34":154,"_36":-5,"_24":25,"_139":-5,"_27":155},{"_44":141,"_32":142,"_30":143,"_132":-5,"_42":43,"_41":144,"_134":25,"_39":145,"_13":146,"_34":20,"_36":147,"_24":25,"_139":-5,"_27":148},{"_44":131,"_32":33,"_30":57,"_132":-5,"_42":43,"_41":133,"_134":59,"_39":135,"_13":136,"_34":137,"_36":138,"_24":25,"_139":-5,"_27":140},"23834182-7ace-11f0-9a73-0224a6c84d84","parentId",37,"latest",["D",1755368622606],25,"http","// Main entry point - redirect to backend\nexport { default } from \"./backend/index.ts\";","label",[33],"3d516cd6-7ad0-11f0-a7a9-0224a6c84d84","README.md","38a799a2-7612-11f0-bec5-0224a6c84d84",52,["D",1755369524888],8,"# Quarter Price Auction Scraper API\n\nA simple, focused JSON API that scrapes **every item** from **every auction** across **all pages** on quarterpriceauction.com.\n\n## Structure\n\n```\n├── backend/\n│ ├── index.ts # Main HTTP endpoint\n│ └── scrapers/\n│ ├── mainScraper.ts # Main orchestration logic\n│ ├── auctionDiscovery.ts # Finds all auction URLs\n│ ├── auctionScraper.ts # Scrapes individual auctions\n│ ├── paginationDetector.ts # Detects page count per auction\n│ └── itemExtractor.ts # Extracts item data from HTML/REDUX\n├── shared/\n│ └── types.ts # TypeScript interfaces\n└── index.ts # Entry point\n```\n\n## Usage\n\n### Main Endpoint\n\n- `GET /` - **Returns ALL items from ALL auctions** (JSON format)\n\n### Other Endpoints\n\n- `GET /csv` - **Returns ALL items from ALL auctions** (CSV format with download)\n- `GET /health` - Health check endpoint\n- `GET /info` - API information and available endpoints\n- `GET /fresh` - Force fresh scrape (no cache)\n- `GET /metadata` - Get metadata and sample items only\n\n## Response Format\n\n### JSON Format (`GET /`)\n\nReturns **all items** from **all auctions**:\n\n```json\n{\n \"timestamp\": \"2025-08-16T18:25:00.000Z\",\n \"totalItems\": 450,\n \"totalAuctions\": 6,\n \"items\": [\n {\n \"title\": \"Google Pixel 9a with Gemini - Unlocked Android Smartphone...\",\n \"auctionTitle\": \"August 22 Auction - Quarter Price\",\n \"auctionUrl\": \"https://www.quarterpriceauction.com/auctions/30135-august-22-auction\",\n \"itemUrl\": \"https://www.quarterpriceauction.com/auctions/30135/lot/291744-google-pixel...\",\n \"imageUrl\": \"https://d3j17a2r8lnfte.cloudfront.net/qrt/2025/8/medium/DG6mWsSZbO4GZPZh7Trv5Fza.jpeg\",\n \"lotNumber\": \"291744\",\n \"currentBid\": \"$45.00\",\n \"startingBid\": \"$1.00\",\n \"bidCount\": 12,\n \"timeLeft\": \"2d 4h 15m\",\n \"category\": \"Electronics\"\n }\n ],\n \"metadata\": {\n \"reduxDataFound\": true,\n \"auctionUrlsFound\": 6,\n \"scrapedAuctions\": 6,\n \"totalPagesScraped\": 45,\n \"processingTimeMs\": 180000\n }\n}\n```\n\n### CSV Format (`GET /csv`)\n\nReturns the same data as a downloadable CSV file with headers:\n\n```csv\nTitle,Auction,Lot Number,Current Bid,Starting Bid,Bid Count,Time Left,Category,Auction URL,Item URL,Image URL,Description\n\"Google Pixel 9a with Gemini - Unlocked Android Smartphone...\",\"August 22 Auction - Quarter Price\",\"291744\",\"$45.00\",\"$1.00\",\"12\",\"2d 4h 15m\",\"Electronics\",\"https://www.quarterpriceauction.com/auctions/30135-august-22-auction\",\"https://www.quarterpriceauction.com/auctions/30135/lot/291744-google-pixel...\",\"https://d3j17a2r8lnfte.cloudfront.net/qrt/2025/8/medium/DG6mWsSZbO4GZPZh7Trv5Fza.jpeg\",\"\"\n```\n\n- **Content-Type**: `text/csv`\n- **Filename**: `quarter-price-auction-items-YYYY-MM-DD.csv`\n- **Download**: Automatically triggers download in browsers\n\n## Features\n\n- **🔍 Comprehensive Scraping**: Extracts every item from every auction\n- **📄 Multi-Page Support**: Automatically detects and handles pagination\n- **🎯 Detailed Item Data**: Titles, lot numbers, prices, images, categories\n- **🧠 Intelligent Parsing**: Uses REDUX data extraction + HTML fallbacks\n- **⚡ Performance Optimized**: High-performance concurrent processing, no artificial delays\n- **🛡️ Error Resilient**: Graceful error handling with detailed metadata\n- **📊 Rich Metadata**: Processing statistics and data source information\n- **📁 Multiple Formats**: JSON and CSV output formats\n\n## Technical Implementation\n\n### Multi-Level Scraping Strategy\n\n1. **Auction Discovery**: Finds all auction URLs from main page\n2. **Pagination Detection**: Uses binary search to find max pages per auction\n3. **Item Extraction**: Tries REDUX data first, falls back to HTML parsing\n4. **Data Enhancement**: Extracts lot numbers, prices, bid counts from containers\n\n### Performance Features\n\n- **Rate Limiting**: 300-500ms delays between requests\n- **Caching**: 10-minute cache for comprehensive scraping\n- **Binary Search**: Efficient pagination detection\n- **Memory Management**: Processes auctions sequentially\n\n### Item Data Fields\n\n- `title` - Full item name/description\n- `auctionTitle` - Which auction the item belongs to\n- `auctionUrl` - Link to the auction page\n- `itemUrl` - Direct link to the individual item/lot\n- `imageUrl` - Product image from CDN\n- `lotNumber` - Auction lot number\n- `currentBid` - Current highest bid amount\n- `startingBid` - Starting bid amount\n- `bidCount` - Number of bids placed\n- `timeLeft` - Time remaining in auction\n- `category` - Item category\n\n## Performance Notes\n\n- **High-Performance Concurrent Processing**: ~2 minutes 15 seconds for 2,956+ items\n- **Concurrent auction processing**: All auctions scraped simultaneously \n- **Concurrent page processing**: All pages within auctions scraped in parallel\n- **Smart pagination detection**: Uses concurrent requests and binary search\n- **No artificial delays**: Optimized for high-performance servers\n- **Throughput**: ~22 items per second, ~0.44 pages per second\n- Results are cached to improve subsequent request performance\n\n## ✅ MISSION\n\n**Goal**: Simple JSON list of every item across all auctions with metadata.\n\n**Status**: ✅ Accomplished - Clean, organized, focused implementation.",[142],"ee869844-7acd-11f0-9f72-0224a6c84d84","shared","70d7061f-95d3-4643-9a03-b479b7308bb0",30,["D",1755368533698],"directory",[150],"f3970530-7acd-11f0-b736-0224a6c84d84","backend","fc20f944-0b9e-4e36-9a6d-25006bf4c3db",31,["D",1755368542201],[157],"pulls","branches",1,"forks","envVars",{"_170":63,"_172":63},"connections",{"_170":63,"_171":63},"count","unfilledCount","emptyCount",{"_44":156,"_32":157,"_30":158,"_132":-5,"_42":43,"_41":159,"_134":59,"_39":253,"_13":63,"_34":154,"_36":-5,"_24":25,"_139":-5,"_186":254,"_27":255},{"_44":249,"_32":217,"_30":210,"_132":158,"_42":43,"_41":159,"_134":59,"_39":250,"_13":63,"_34":154,"_36":-5,"_24":25,"_139":-5,"_186":251,"_27":252},{"_44":242,"_32":243,"_30":244,"_132":210,"_42":43,"_41":159,"_134":59,"_39":245,"_13":63,"_34":35,"_36":246,"_24":25,"_139":-5,"_186":247,"_27":248},{"_44":234,"_32":235,"_30":236,"_132":210,"_42":43,"_41":237,"_134":25,"_39":238,"_13":213,"_34":35,"_36":239,"_24":25,"_139":-5,"_186":240,"_27":241},{"_44":226,"_32":227,"_30":228,"_132":210,"_42":43,"_41":229,"_134":25,"_39":230,"_13":63,"_34":35,"_36":231,"_24":25,"_139":-5,"_186":232,"_27":233},{"_44":218,"_32":219,"_30":220,"_132":210,"_42":43,"_41":221,"_134":59,"_39":222,"_13":164,"_34":35,"_36":223,"_24":25,"_139":-5,"_186":224,"_27":225},{"_44":207,"_32":208,"_30":209,"_132":210,"_42":43,"_41":211,"_134":59,"_39":212,"_13":213,"_34":35,"_36":214,"_24":25,"_139":-5,"_186":215,"_27":216},{"_44":202,"_32":33,"_30":31,"_132":158,"_42":43,"_41":14,"_134":25,"_39":203,"_13":204,"_34":35,"_36":38,"_24":25,"_139":-5,"_186":205,"_27":206},{"_44":149,"_32":150,"_30":151,"_132":-5,"_42":43,"_41":152,"_134":59,"_39":199,"_13":63,"_34":154,"_36":-5,"_24":25,"_139":-5,"_186":200,"_27":201},{"_44":192,"_32":193,"_30":194,"_132":151,"_42":43,"_41":152,"_134":59,"_39":195,"_13":63,"_34":35,"_36":196,"_24":25,"_139":-5,"_186":197,"_27":198},{"_44":141,"_32":142,"_30":143,"_132":-5,"_42":43,"_41":144,"_134":25,"_39":189,"_13":146,"_34":20,"_36":147,"_24":25,"_139":-5,"_186":190,"_27":191},{"_44":131,"_32":33,"_30":57,"_132":-5,"_42":43,"_41":133,"_134":59,"_39":185,"_13":136,"_34":137,"_36":138,"_24":25,"_139":-5,"_186":187,"_27":188},["D",1755368622606],"children",[],[33],["D",1755369524888],[],[142],"ee868ad4-7acd-11f0-9f72-0224a6c84d84","types.ts","ee868bc4-7acd-11f0-9f72-0224a6c84d84",["D",1755368533698],"export interface AuctionItem {\n title: string;\n currentBid?: string;\n timeLeft?: string;\n imageUrl?: string;\n auctionUrl?: string;\n itemUrl?: string;\n description?: string;\n lotNumber?: string;\n auctionTitle?: string;\n startingBid?: string;\n bidCount?: number;\n category?: string;\n}\n\nexport interface ScrapedData {\n timestamp: string;\n totalItems: number;\n totalAuctions: number;\n items: AuctionItem[];\n error?: string;\n metadata?: {\n reduxDataFound: boolean;\n auctionUrlsFound: number;\n scrapedAuctions: number;\n totalPagesScraped: number;\n processingTimeMs: number;\n };\n}",[],[150,193],["D",1755368533698],[182],[150],"411ce624-7ad0-11f0-a713-0224a6c84d84",["D",1755369531261],6,[],[157,33],"fe273e56-7ace-11f0-a0e5-0224a6c84d84","paginationDetector.ts","f964cf56-7acd-11f0-9fe5-0224a6c84d84","31dd3db8-c538-401b-86da-04105849d3b1",43,["D",1755368989420],2,"/**\n * Detects the maximum number of pages for an auction\n */\nexport async function detectMaxPages(auctionUrl: string, firstPageHtml: string, itemCount: number): Promise {\n let maxPage = 1;\n \n // If we found close to 50 items on page 1, there are likely more pages\n if (itemCount >= 45) {\n console.log(`${itemCount} items found on page 1, checking for more pages...`);\n \n try {\n // Test if page 2 exists\n const testPage2 = await fetch(`${auctionUrl}?page=2`);\n if (testPage2.ok) {\n const testHtml = await testPage2.text();\n // Check if page 2 has content\n if (testHtml.length > 100000 && testHtml.includes('lot-img')) {\n console.log(`Page 2 exists for ${auctionUrl}, finding max page...`);\n \n // Use concurrent requests to find max page more efficiently\n maxPage = await findMaxPageConcurrently(auctionUrl);\n console.log(`Found ${maxPage} pages for ${auctionUrl}`);\n }\n }\n } catch (e) {\n console.log(`Error detecting pages for ${auctionUrl}:`, e);\n }\n }\n \n return maxPage;\n}\n\n/**\n * Finds max page using concurrent requests for faster detection\n */\nasync function findMaxPageConcurrently(auctionUrl: string): Promise {\n // Test multiple pages concurrently to find the boundary faster\n const testPages = [5, 10, 15, 20, 25, 30];\n \n const pagePromises = testPages.map(async (page) => {\n try {\n const response = await fetch(`${auctionUrl}?page=${page}`);\n if (response.ok) {\n const html = await response.text();\n const isValid = html.length > 100000 && html.includes('lot-img');\n return { page, isValid };\n }\n return { page, isValid: false };\n } catch (e) {\n return { page, isValid: false };\n }\n });\n \n const results = await Promise.all(pagePromises);\n \n // Find the highest valid page\n let maxValidPage = 1;\n for (const result of results) {\n if (result.isValid && result.page > maxValidPage) {\n maxValidPage = result.page;\n }\n }\n \n // If we found a high valid page, do a more precise binary search in that range\n if (maxValidPage > 1) {\n const preciseMax = await binarySearchMaxPage(auctionUrl, maxValidPage, Math.min(maxValidPage + 10, 50));\n return preciseMax;\n }\n \n return maxValidPage;\n}\n\n/**\n * Binary search for exact max page in a smaller range\n */\nasync function binarySearchMaxPage(auctionUrl: string, low: number, high: number): Promise {\n let lastValidPage = low;\n \n while (low <= high) {\n const mid = Math.floor((low + high) / 2);\n try {\n const testResponse = await fetch(`${auctionUrl}?page=${mid}`);\n if (testResponse.ok) {\n const testHtml = await testResponse.text();\n if (testHtml.length > 100000 && testHtml.includes('lot-img')) {\n lastValidPage = mid;\n low = mid + 1;\n } else {\n high = mid - 1;\n }\n } else {\n high = mid - 1;\n }\n } catch (e) {\n high = mid - 1;\n }\n }\n \n return lastValidPage;\n}",[],[157,217,208],"scrapers","f56fe204-7ace-11f0-b506-0224a6c84d84","mainScraper.ts","1ce292d8-7ace-11f0-a9b5-0224a6c84d84",42,["D",1755368974797],"import type { ScrapedData } from \"../../shared/types.ts\";\nimport { discoverAuctionUrls } from \"./auctionDiscovery.ts\";\nimport { scrapeAuctionItems } from \"./auctionScraper.ts\";\n\n/**\n * Main scraper function - gets all items from all auctions\n */\nexport async function scrapeAllAuctionItems(): Promise {\n const startTime = Date.now();\n \n try {\n // Discover all auction URLs\n const auctionUrls = await discoverAuctionUrls();\n \n if (auctionUrls.length === 0) {\n return {\n timestamp: new Date().toISOString(),\n totalItems: 0,\n totalAuctions: 0,\n items: [],\n error: \"No auction URLs found\"\n };\n }\n \n const allItems = [];\n let totalPagesScraped = 0;\n \n // Process auctions concurrently\n const auctionPromises = auctionUrls.map(async (auctionUrl) => {\n try {\n console.log(`Processing auction: ${auctionUrl}`);\n const auctionResult = await scrapeAuctionItems(auctionUrl);\n console.log(`Completed auction ${auctionUrl}: ${auctionResult.items.length} items, ${auctionResult.pagesScraped} pages`);\n return auctionResult;\n } catch (error) {\n console.log(`Failed to process auction ${auctionUrl}:`, error);\n return { items: [], pagesScraped: 0 };\n }\n });\n \n // Wait for all auctions to complete\n const auctionResults = await Promise.all(auctionPromises);\n \n // Combine results\n for (const result of auctionResults) {\n allItems.push(...result.items);\n totalPagesScraped += result.pagesScraped;\n }\n \n const processingTime = Date.now() - startTime;\n \n return {\n timestamp: new Date().toISOString(),\n totalItems: allItems.length,\n totalAuctions: auctionUrls.length,\n items: allItems,\n metadata: {\n reduxDataFound: true, // We always try REDUX first\n auctionUrlsFound: auctionUrls.length,\n scrapedAuctions: auctionUrls.length,\n totalPagesScraped,\n processingTimeMs: processingTime\n }\n };\n \n } catch (error) {\n console.error(\"Scraping error:\", error);\n return {\n timestamp: new Date().toISOString(),\n totalItems: 0,\n totalAuctions: 0,\n items: [],\n error: error instanceof Error ? error.message : \"Unknown error occurred\"\n };\n }\n}",[],[157,217,219],"0be422f8-7ace-11f0-b25a-0224a6c84d84","itemExtractor.ts","0be423e8-7ace-11f0-b25a-0224a6c84d84",33,["D",1755368582978],"import type { AuctionItem } from \"../../shared/types.ts\";\n\n/**\n * Extracts items from REDUX data\n */\nexport function extractItemsFromReduxData(reduxData: any, auctionTitle: string, auctionUrl: string): AuctionItem[] {\n const items: AuctionItem[] = [];\n \n // Look for lots/items in various possible locations in the redux state\n const possibleItemSources = [\n reduxData.lots,\n reduxData.items,\n reduxData.auction?.lots,\n reduxData.auction?.items,\n reduxData.lotList,\n reduxData.currentAuction?.lots,\n reduxData.auctionLots\n ];\n \n for (const source of possibleItemSources) {\n if (Array.isArray(source)) {\n for (const item of source) {\n if (item && typeof item === 'object') {\n const auctionItem = createAuctionItem(item, auctionTitle, auctionUrl);\n items.push(auctionItem);\n }\n }\n } else if (source && typeof source === 'object') {\n // Handle object with item IDs as keys\n for (const [key, item] of Object.entries(source)) {\n if (item && typeof item === 'object') {\n const auctionItem = createAuctionItem(item as any, auctionTitle, auctionUrl, key);\n items.push(auctionItem);\n }\n }\n }\n }\n \n return items;\n}\n\n/**\n * Extracts items from HTML\n */\nexport function extractItemsFromHtml(html: string, auctionTitle: string, auctionUrl: string): AuctionItem[] {\n const items: AuctionItem[] = [];\n \n // Look for images with class=\"lot-img\" which contain the item titles in alt text\n const lotImages = html.match(/]*class[^>]*lot-img[^>]*>/gi) || [];\n console.log(`Found ${lotImages.length} lot images`);\n \n for (const imgTag of lotImages) {\n const item: AuctionItem = {\n title: 'Unknown Item',\n auctionTitle,\n auctionUrl\n };\n \n // Extract title from alt attribute\n const altMatch = imgTag.match(/alt=[\"']([^\"']+)[\"']/i);\n if (altMatch && altMatch[1]) {\n item.title = altMatch[1].replace(/&/g, '&').trim();\n }\n \n // Extract image URL\n const srcMatch = imgTag.match(/src=[\"']([^\"']+)[\"']/i);\n if (srcMatch && srcMatch[1]) {\n item.imageUrl = srcMatch[1];\n }\n \n items.push(item);\n }\n \n // Enhance items with additional data from lot containers\n enhanceItemsFromContainers(html, items, auctionTitle, auctionUrl);\n \n // Remove duplicates and items without meaningful data\n const uniqueItems = items.filter((item, index, self) => {\n const isDuplicate = index !== self.findIndex(other => other.title === item.title);\n if (isDuplicate) return false;\n return item.title !== 'Unknown Item' && item.title.length > 3;\n });\n \n console.log(`Extracted ${uniqueItems.length} unique items from HTML`);\n return uniqueItems;\n}\n\n/**\n * Creates an AuctionItem from raw data\n */\nfunction createAuctionItem(itemData: any, auctionTitle: string, auctionUrl: string, key?: string): AuctionItem {\n const auctionItem: AuctionItem = {\n title: itemData.title || itemData.name || itemData.description || `Item ${key || 'Unknown'}`,\n auctionTitle,\n auctionUrl\n };\n \n if (itemData.current_bid !== undefined) auctionItem.currentBid = `${itemData.current_bid}`;\n if (itemData.starting_bid !== undefined) auctionItem.startingBid = `${itemData.starting_bid}`;\n if (itemData.bid_count !== undefined) auctionItem.bidCount = itemData.bid_count;\n if (itemData.time_left) auctionItem.timeLeft = itemData.time_left;\n if (itemData.lot_number) auctionItem.lotNumber = itemData.lot_number.toString();\n if (itemData.category) auctionItem.category = itemData.category;\n if (itemData.description && itemData.description !== auctionItem.title) {\n auctionItem.description = itemData.description;\n }\n if (itemData.image_url || itemData.imageUrl) {\n auctionItem.imageUrl = itemData.image_url || itemData.imageUrl;\n }\n if (itemData.url || itemData.lot_url) {\n const itemUrl = itemData.url || itemData.lot_url;\n auctionItem.itemUrl = itemUrl.startsWith('/') ? \n `https://www.quarterpriceauction.com${itemUrl}` : itemUrl;\n }\n \n return auctionItem;\n}\n\n/**\n * Enhances items with additional data from lot containers\n */\nfunction enhanceItemsFromContainers(html: string, items: AuctionItem[], auctionTitle: string, auctionUrl: string): void {\n const lotContainerPatterns = [\n /]*class[^>]*lot[^>]*>[\\s\\S]*?<\\/div>/gi,\n /]*class[^>]*lotList[^>]*>[\\s\\S]*?<\\/div>/gi\n ];\n \n for (const pattern of lotContainerPatterns) {\n const matches = html.match(pattern) || [];\n \n for (const lotHtml of matches.slice(0, 200)) {\n if (lotHtml.length < 100) continue;\n \n const containerImages = lotHtml.match(/]*class[^>]*lot-img[^>]*>/gi) || [];\n if (containerImages.length === 0) continue;\n \n const imgTag = containerImages[0];\n const altMatch = imgTag.match(/alt=[\"']([^\"']+)[\"']/i);\n if (!altMatch) continue;\n \n const title = altMatch[1].replace(/&/g, '&').trim();\n let existingItem = items.find(item => item.title === title);\n \n if (!existingItem) {\n existingItem = {\n title,\n auctionTitle,\n auctionUrl\n };\n \n const srcMatch = imgTag.match(/src=[\"']([^\"']+)[\"']/i);\n if (srcMatch && srcMatch[1]) {\n existingItem.imageUrl = srcMatch[1];\n }\n \n items.push(existingItem);\n }\n \n // Extract additional data\n enhanceItemWithContainerData(existingItem, lotHtml);\n }\n }\n}\n\n/**\n * Enhances a single item with data from its container HTML\n */\nfunction enhanceItemWithContainerData(item: AuctionItem, containerHtml: string): void {\n // Look for prices\n const priceMatches = containerHtml.match(/\\$[\\d,]+\\.?\\d*/g) || [];\n if (priceMatches.length > 0 && !item.currentBid) {\n item.currentBid = priceMatches[0];\n }\n \n // Look for lot numbers\n const lotNumberPatterns = [\n containerHtml.match(/lot[^0-9]*(\\d+)/i),\n containerHtml.match(/item[^0-9]*(\\d+)/i),\n containerHtml.match(/#(\\d+)/),\n containerHtml.match(/data-lot[^0-9]*(\\d+)/i)\n ];\n \n for (const pattern of lotNumberPatterns) {\n if (pattern && pattern[1] && !item.lotNumber) {\n item.lotNumber = pattern[1];\n break;\n }\n }\n \n // Look for links to individual lot pages\n const linkMatch = containerHtml.match(/]*href=[\"']([^\"']+)[\"']/i);\n if (linkMatch && linkMatch[1] && !item.itemUrl) {\n let itemUrl = linkMatch[1];\n if (itemUrl.startsWith('/')) {\n itemUrl = 'https://www.quarterpriceauction.com' + itemUrl;\n }\n if (itemUrl.includes('lot') || itemUrl.includes('item')) {\n item.itemUrl = itemUrl;\n }\n }\n \n // Look for bid count\n const bidCountMatch = containerHtml.match(/(\\d+)\\s*bids?/i);\n if (bidCountMatch && !item.bidCount) {\n item.bidCount = parseInt(bidCountMatch[1]);\n }\n \n // Look for time left\n const timeLeftMatch = containerHtml.match(/(\\d+[dhms]|\\d+:\\d+:\\d+|\\d+\\s*(?:days?|hours?|minutes?|mins?))/i);\n if (timeLeftMatch && !item.timeLeft) {\n item.timeLeft = timeLeftMatch[1];\n }\n}",[],[157,217,227],"f074bcc0-7ace-11f0-b3b3-0224a6c84d84","auctionScraper.ts","16820a4a-7ace-11f0-90f1-0224a6c84d84",41,["D",1755368966439],"import type { AuctionItem } from \"../../shared/types.ts\";\nimport { extractItemsFromReduxData, extractItemsFromHtml } from \"./itemExtractor.ts\";\nimport { detectMaxPages } from \"./paginationDetector.ts\";\n\n/**\n * Scrapes all items from a single auction across all pages\n */\nexport async function scrapeAuctionItems(auctionUrl: string): Promise<{items: AuctionItem[], pagesScraped: number}> {\n const items: AuctionItem[] = [];\n let pagesScraped = 0;\n \n try {\n console.log(`Starting to scrape auction: ${auctionUrl}`);\n \n // Get the first page\n const response = await fetch(auctionUrl);\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n \n const html = await response.text();\n pagesScraped++;\n \n // Extract auction title\n const auctionTitleMatch = html.match(/]*>([^<]+)<\\/title>/i);\n const auctionTitle = auctionTitleMatch ? \n auctionTitleMatch[1].replace(/Quarter Price.*?-\\s*/, '').trim() : \n auctionUrl.split('/').pop()?.replace(/-/g, ' ') || 'Unknown Auction';\n \n // Extract items from the first page\n let pageItems = extractItemsFromPage(html, auctionTitle, auctionUrl);\n items.push(...pageItems);\n \n console.log(`Found ${pageItems.length} items on page 1 of ${auctionUrl}`);\n \n // Detect how many pages this auction has\n const maxPage = await detectMaxPages(auctionUrl, html, pageItems.length);\n \n // Scrape remaining pages concurrently\n if (maxPage > 1) {\n const pagePromises = [];\n \n for (let page = 2; page <= Math.min(maxPage, 50); page++) {\n const pagePromise = scrapePageConcurrently(auctionUrl, page, auctionTitle);\n pagePromises.push(pagePromise);\n }\n \n // Wait for all pages to complete\n const pageResults = await Promise.allSettled(pagePromises);\n \n // Process results\n for (let i = 0; i < pageResults.length; i++) {\n const page = i + 2; // Pages start at 2\n const result = pageResults[i];\n \n if (result.status === 'fulfilled') {\n const { items: pageItems, success } = result.value;\n if (success && pageItems.length > 0) {\n items.push(...pageItems);\n pagesScraped++;\n console.log(`Found ${pageItems.length} items on page ${page} of ${auctionUrl}`);\n } else if (!success) {\n console.log(`Page ${page} appears to be empty or invalid`);\n }\n } else {\n console.log(`Error scraping page ${page} of ${auctionUrl}:`, result.reason);\n }\n }\n }\n \n console.log(`Completed scraping ${auctionUrl}: ${items.length} total items across ${pagesScraped} pages`);\n \n } catch (error) {\n console.log(`Error scraping auction ${auctionUrl}:`, error);\n }\n \n return { items, pagesScraped };\n}\n\n/**\n * Extracts items from a single page\n */\nfunction extractItemsFromPage(html: string, auctionTitle: string, auctionUrl: string): AuctionItem[] {\n // Try REDUX data first\n const reduxMatch = html.match(/window\\.REDUX_DATA\\s*=\\s*({.*?});/s);\n if (reduxMatch) {\n try {\n const reduxData = JSON.parse(reduxMatch[1]);\n const reduxItems = extractItemsFromReduxData(reduxData, auctionTitle, auctionUrl);\n if (reduxItems.length > 0) {\n return reduxItems;\n }\n } catch (e) {\n console.log(\"Failed to parse REDUX data, falling back to HTML\");\n }\n }\n \n // Fall back to HTML parsing\n return extractItemsFromHtml(html, auctionTitle, auctionUrl);\n}\n\n/**\n * Checks if a page has valid content\n */\nfunction isValidPage(html: string): boolean {\n return html.length > 100000 && \n html.includes('lot-img') && \n !html.includes('No items found') && \n !html.includes('No lots found');\n}\n\n/**\n * Scrapes a single page concurrently\n */\nasync function scrapePageConcurrently(auctionUrl: string, page: number, auctionTitle: string): Promise<{items: AuctionItem[], success: boolean}> {\n try {\n const pageUrl = `${auctionUrl}?page=${page}`;\n \n const pageResponse = await fetch(pageUrl);\n if (!pageResponse.ok) {\n return { items: [], success: false };\n }\n \n const pageHtml = await pageResponse.text();\n \n // Check if page has content\n if (!isValidPage(pageHtml)) {\n return { items: [], success: false };\n }\n \n // Extract items from this page\n const pageItems = extractItemsFromPage(pageHtml, auctionTitle, auctionUrl);\n \n return { items: pageItems, success: pageItems.length > 0 };\n \n } catch (error) {\n console.log(`Error scraping page ${page} of ${auctionUrl}:`, error);\n return { items: [], success: false };\n }\n}",[],[157,217,235],"f396fc16-7acd-11f0-b736-0224a6c84d84","auctionDiscovery.ts","f396fd06-7acd-11f0-b736-0224a6c84d84",["D",1755368542201],"/**\n * Discovers all auction URLs from the main page\n */\nexport async function discoverAuctionUrls(): Promise {\n try {\n const response = await fetch(\"https://www.quarterpriceauction.com/\");\n \n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n \n const html = await response.text();\n \n // Extract auction URLs from the HTML\n const auctionUrls = [...new Set((html.match(/href=[\"'][^\"']*\\/auctions\\/[^\"']*[\"']/g) || [])\n .map(m => m.replace(/href=[\"']/, '').replace(/[\"']$/, ''))\n .filter(url => url.includes('/auctions/') && !url.includes('#'))\n .map(url => url.startsWith('/') ? `https://www.quarterpriceauction.com${url}` : url))];\n \n console.log(`Found ${auctionUrls.length} auction URLs`);\n return auctionUrls;\n \n } catch (error) {\n console.error(\"Error discovering auction URLs:\", error);\n return [];\n }\n}",[],[157,217,243],"f397072e-7acd-11f0-b736-0224a6c84d84",["D",1755368542201],[175,176,177,178,179],[157,217],["D",1755368542201],[174,180],[157],["D",1754848079739],{"_97":96,"_104":100},{"_97":96,"_98":-5,"_99":100,"_18":55,"_101":102,"_103":25},{"_13":86,"_32":87,"_44":43,"_88":45,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":46,"_95":96},"ctx",{"_79":-5,"_124":293,"_294":295,"_296":297,"_298":25,"_299":25,"_300":25,"_301":59,"_302":-5,"_303":-7,"_304":-7,"_305":63,"_306":25,"_307":25,"_308":-5,"_309":25,"_310":25,"_311":25,"_312":-5},"ENV",{"_264":265,"_266":267,"_268":87,"_269":270,"_271":272,"_273":274,"_275":276,"_277":278,"_279":280,"_281":282,"_283":284,"_285":286,"_287":288,"_289":290,"_291":292},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v53