FanFic Scraper

\n );\n}\n\nfunction client() {\n console.log(\"Initializing client-side React application\");\n createRoot(document.getElementById(\"root\")).render();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nasync function scrapePage(url) {\n console.log(`Starting to scrape page: ${url}`);\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n\n // We'll keep this part as is\n const threadId = url.split(\"/\").pop().split(\".\")[0];\n const threadmarkCategories = [19, 10, 13, 16];\n const threadmarkResults = await Promise.all(threadmarkCategories.map(async (category) => {\n const threadmarkUrl = `${url}/threadmarks?hide_wrapper=1&display=embedded&threadmark_category=${category}`;\n const response = await fetch(\n `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(threadmarkUrl)\n }&render_js=true&premium_proxy=true&stealth_proxy=true`,\n );\n if (!response.ok) {\n console.log(`Error fetching threadmarks for category ${category}`);\n return null;\n }\n // We'll log the full response text here\n const responseText = await response.text();\n console.log(`Full response for category ${category}:`, responseText);\n try {\n return JSON.parse(responseText);\n } catch (error) {\n console.log(`Error parsing JSON for category ${category}:`, error);\n return null;\n }\n }));\n\n // The rest of the function remains the same\n const extractRules = {\n title: \"h1\",\n articles: {\n selector: \".message:has(.message-cell--threadmark-header)\",\n type: \"list\",\n output: {\n threadmark: \".threadmarkLabel\",\n date: {\n selector: \".message-attribution-main time\",\n output: \"text\",\n },\n author: {\n selector: \".message-attribution-main .username\",\n output: \"text\",\n },\n },\n },\n };\n\n console.log(`Sending request to ScrapingBee for URL: ${url} at ${new Date().toISOString()}`);\n const startTime = Date.now();\n try {\n const requestUrl = `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(url)\n }&render_js=true&premium_proxy=true&stealth_proxy=true&extract_rules=${\n encodeURIComponent(JSON.stringify(extractRules))\n }`;\n\n const response = await fetch(requestUrl, {\n method: \"GET\",\n });\n\n const duration = Date.now() - startTime;\n console.log(`Received response from ScrapingBee at ${new Date().toISOString()}. Duration: ${duration}ms`);\n console.log(`Received response from ScrapingBee for URL: ${url}. Status: ${response.status}`);\n\n if (!response.ok) {\n console.log(`Error response from ScrapingBee. Status: ${response.status}`);\n const errorBody = await response.text();\n console.log(`Error response body: ${errorBody}`);\n console.log(`Request headers:`, Object.fromEntries(response.headers));\n throw new Error(`HTTP error! status: ${response.status}, body: ${errorBody}`);\n }\n\n const responseText = await response.text();\n console.log(`Response size (in characters): ${responseText.length}`);\n console.log(`Full response:`, responseText);\n\n let result;\n try {\n result = JSON.parse(responseText);\n } catch (error) {\n console.log(`Error parsing JSON:`, error);\n console.log(`Response was not valid JSON:`, responseText);\n throw new Error(`Failed to parse response as JSON`);\n }\n\n console.log(`Title scraped: ${result.title}`);\n console.log(`Number of articles scraped: ${result.articles ? result.articles.length : 0}`);\n\n if (result.articles && result.articles.length > 0) {\n console.log(`Detected ${result.articles.length} threadmarked articles on this page.`);\n result.articles.forEach((article, index) => {\n console.log(`Threadmark ${index + 1}: ${article.threadmark}, Date: ${article.date}, Author: ${article.author}`);\n });\n } else {\n console.log(`No articles were scraped or no threadmarks detected.`);\n }\n\n return {\n ...result,\n threadmarks: threadmarkResults.filter(Boolean),\n };\n } catch (error) {\n console.log(`Error occurred while scraping URL: ${url}`);\n console.log(`Error details: ${error.message}`);\n console.log(`Error stack: ${error.stack}`);\n throw error;\n }\n}\n\nasync function getApiKey() {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n return apiKey;\n}\n\nasync function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/scrape\") {\n try {\n const { url } = await request.json();\n console.log(`Processing scrape request for URL: ${url}`);\n const result = await scrapePage(url);\n console.log(`Successfully scraped URL: ${url}`);\n return new Response(JSON.stringify(result), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.log(`Error occurred while processing request. Error details: ${error.message}`);\n console.log(`Error stack trace: ${error.stack}`);\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n }\n\n console.log(\"Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Web Scraper\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst css = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"] {\n width: 70%;\n padding: 10px;\n margin-right: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;\n\nexport default server;","deleted","label","children",[],[153],"files",[151],"counts",{"_174":31,"_175":176,"_177":176,"_178":31},"pulls","branches",1,"forks","envVars","contents",[181],{"_107":152,"_100":153,"_154":120,"_155":-5,"_156":140,"_157":139,"_158":113,"_109":182,"_104":160,"_161":162,"_163":164,"_165":27,"_166":-5,"_103":183},["D",1727186318648],[153],"file","upstreamProject",{"_107":187,"_100":188,"_116":-5,"_117":-5,"_114":115,"_21":189},"9dbd1426-27b0-11f0-aecd-569c3dd06744","FanFicScraper",{"_98":99,"_135":136},"upstreamBranch",{"_107":143,"_109":192,"_193":194,"_100":42,"_141":187,"_104":195,"_142":-5,"_144":-5,"_196":197,"_111":-5,"_133":27,"_114":115,"_148":99,"_147":188},["D",1746229996260],"updatedAt",["D",1727710402039],143,"lockfileKey","1746250683190-esm.town_v_willthereader_FanFicScraper@143-main","emptyEnvVars","isNew","routes/_app.x.$handle.$name._index",{"_98":99,"_100":101,"_102":-7,"_104":-7,"_105":202,"_179":206,"_209":-7,"_210":211,"_214":-5,"_170":215,"_219":220},{"_107":108,"_100":101,"_109":203,"_111":-5,"_112":113,"_114":115,"_116":-5,"_117":-5,"_12":118,"_119":120,"_121":113,"_122":-5,"_123":31,"_124":125,"_126":125,"_127":128,"_129":128,"_130":128,"_131":125,"_132":125,"_133":27,"_21":204,"_137":205},["D",1746229994130],{"_98":99,"_135":136},{"_104":139,"_100":42,"_107":140,"_141":108,"_142":143,"_144":31,"_145":115,"_146":118,"_147":101,"_148":99},[207],{"_107":152,"_100":153,"_154":120,"_155":-5,"_156":140,"_157":139,"_158":113,"_109":182,"_104":160,"_161":162,"_163":164,"_165":27,"_166":-5,"_103":208},[153],"readme","upstream",{"_107":143,"_109":212,"_193":213,"_100":42,"_141":187,"_104":195,"_142":-5,"_144":-5,"_196":197,"_111":-5,"_133":27,"_114":115,"_148":99,"_147":188},["D",1746229996260],["D",1727710402039],"pull",[216],{"_107":152,"_100":153,"_154":120,"_155":-5,"_156":140,"_157":139,"_158":113,"_109":159,"_104":160,"_161":162,"_163":164,"_165":27,"_166":-5,"_167":217,"_103":218},[],[153],"previews",[216],"routes/_app","actionData","errors"]

FanFic Scraper

Result:

{result.title}

Articles:

Threadmarks:

Threadmark: {article.threadmark}