FanFic Scraper

\n );\n}\n\nfunction client() {\n console.log(\"Initializing client-side React application\");\n createRoot(document.getElementById(\"root\")).render();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nasync function scrapePage(url) {\n console.log(`Starting to scrape page: ${url}`);\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n\n // We'll keep this part as is\n const threadId = url.split(\"/\").pop().split(\".\")[0];\n const threadmarkCategories = [19, 10, 13, 16];\n const threadmarkResults = await Promise.all(threadmarkCategories.map(async (category) => {\n const threadmarkUrl = `${url}/threadmarks?hide_wrapper=1&display=embedded&threadmark_category=${category}`;\n const response = await fetch(\n `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(threadmarkUrl)\n }&render_js=true&premium_proxy=true&stealth_proxy=true`,\n );\n if (!response.ok) {\n console.log(`Error fetching threadmarks for category ${category}`);\n return null;\n }\n // We'll log the full response text here\n const responseText = await response.text();\n console.log(`Full response for category ${category}:`, responseText);\n try {\n return JSON.parse(responseText);\n } catch (error) {\n console.log(`Error parsing JSON for category ${category}:`, error);\n return null;\n }\n }));\n\n // The rest of the function remains the same\n const extractRules = {\n title: \"h1\",\n articles: {\n selector: \".message:has(.message-cell--threadmark-header)\",\n type: \"list\",\n output: {\n threadmark: \".threadmarkLabel\",\n date: {\n selector: \".message-attribution-main time\",\n output: \"text\",\n },\n author: {\n selector: \".message-attribution-main .username\",\n output: \"text\",\n },\n },\n },\n };\n\n console.log(`Sending request to ScrapingBee for URL: ${url} at ${new Date().toISOString()}`);\n const startTime = Date.now();\n try {\n const requestUrl = `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(url)\n }&render_js=true&premium_proxy=true&stealth_proxy=true&extract_rules=${\n encodeURIComponent(JSON.stringify(extractRules))\n }`;\n\n const response = await fetch(requestUrl, {\n method: \"GET\",\n });\n\n const duration = Date.now() - startTime;\n console.log(`Received response from ScrapingBee at ${new Date().toISOString()}. Duration: ${duration}ms`);\n console.log(`Received response from ScrapingBee for URL: ${url}. Status: ${response.status}`);\n\n if (!response.ok) {\n console.log(`Error response from ScrapingBee. Status: ${response.status}`);\n const errorBody = await response.text();\n console.log(`Error response body: ${errorBody}`);\n console.log(`Request headers:`, Object.fromEntries(response.headers));\n throw new Error(`HTTP error! status: ${response.status}, body: ${errorBody}`);\n }\n\n const responseText = await response.text();\n console.log(`Response size (in characters): ${responseText.length}`);\n console.log(`Full response:`, responseText);\n\n let result;\n try {\n result = JSON.parse(responseText);\n } catch (error) {\n console.log(`Error parsing JSON:`, error);\n console.log(`Response was not valid JSON:`, responseText);\n throw new Error(`Failed to parse response as JSON`);\n }\n\n console.log(`Title scraped: ${result.title}`);\n console.log(`Number of articles scraped: ${result.articles ? result.articles.length : 0}`);\n\n if (result.articles && result.articles.length > 0) {\n console.log(`Detected ${result.articles.length} threadmarked articles on this page.`);\n result.articles.forEach((article, index) => {\n console.log(`Threadmark ${index + 1}: ${article.threadmark}, Date: ${article.date}, Author: ${article.author}`);\n });\n } else {\n console.log(`No articles were scraped or no threadmarks detected.`);\n }\n\n return {\n ...result,\n threadmarks: threadmarkResults.filter(Boolean),\n };\n } catch (error) {\n console.log(`Error occurred while scraping URL: ${url}`);\n console.log(`Error details: ${error.message}`);\n console.log(`Error stack: ${error.stack}`);\n throw error;\n }\n}\n\nasync function getApiKey() {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n return apiKey;\n}\n\nasync function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/scrape\") {\n try {\n const { url } = await request.json();\n console.log(`Processing scrape request for URL: ${url}`);\n const result = await scrapePage(url);\n console.log(`Successfully scraped URL: ${url}`);\n return new Response(JSON.stringify(result), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.log(`Error occurred while processing request. Error details: ${error.message}`);\n console.log(`Error stack trace: ${error.stack}`);\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n }\n\n console.log(\"Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Web Scraper\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst css = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"] {\n width: 70%;\n padding: 10px;\n margin-right: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;\n\nexport default server;","deleted","label",[99],"permalink","https://www.val.town/x/willthereader/broadRedMarmoset/code/main.tsx","extension","actionData","errors"]

FanFic Scraper

Result:

{result.title}

Articles:

Threadmarks:

Threadmark: {article.threadmark}