FanFic Scraper

\n setUrl(e.target.value)}\n placeholder=\"Enter URL to scrape\"\n required\n />\n \n \n {result && (\n

Result:

{result.title}

Content:

Chapters:

\n
Threadmark: {article.threadmark}
\n
Date: {article.date}
\n
Author: {article.author}
\n
\n

\n\n {result.error &&

Error: {result.error}

}\n

\n )}\n

\n );\n}\n\nfunction client() {\n console.log(\"Initializing client-side React application\");\n createRoot(document.getElementById(\"root\")).render();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nasync function scrapePage(url) {\n console.log(`Starting to scrape page: ${url}`);\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n\n const extractRules = {\n title: \"h1\",\n articles: {\n selector: \".message:has(.message-cell--threadmark-header)\",\n type: \"list\",\n output: {\n threadmark: \".threadmarkLabel\",\n date: {\n selector: \".message-attribution-main time\",\n output: \"text\",\n },\n author: {\n selector: \".message-attribution-main .username\",\n output: \"text\",\n },\n },\n },\n };\n\n console.log(`Sending request to ScrapingBee for URL: ${url} at ${new Date().toISOString()}`);\n const startTime = Date.now();\n try {\n const response = await fetch(\n `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(url)\n }&render_js=false&extract_rules=${encodeURIComponent(JSON.stringify(extractRules))}`,\n );\n\n const duration = Date.now() - startTime;\n console.log(`Received response from ScrapingBee at ${new Date().toISOString()}. Duration: ${duration}ms`);\n console.log(`Received response from ScrapingBee for URL: ${url}. Status: ${response.status}`);\n\n if (!response.ok) {\n console.log(`Error response from ScrapingBee. Status: ${response.status}`);\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n\n // Log response size before parsing\n const responseText = await response.text();\n console.log(`Response size (in characters): ${responseText.length}`);\n\n const result = JSON.parse(responseText);\n console.log(`Title scraped: ${result.title}`);\n console.log(`Number of articles scraped: ${result.articles ? result.articles.length : 0}`);\n\n // Log all detected threadmarks\n if (result.articles && result.articles.length > 0) {\n console.log(`Detected ${result.articles.length} threadmarked articles on this page.`);\n result.articles.forEach((article, index) => {\n console.log(`Threadmark ${index + 1}: ${article.threadmark}, Date: ${article.date}, Author: ${article.author}`);\n });\n } else {\n console.log(`No articles were scraped or no threadmarks detected.`);\n }\n\n // Log extracted data for edge cases\n if (result.articles && result.articles.length > 0) {\n result.articles.forEach((article, index) => {\n console.log(`Threadmark ${index + 1}: ${article.threadmark || \"N/A\"}`);\n console.log(`Date: ${article.date || \"N/A\"}`);\n console.log(`Author: ${article.author || \"N/A\"}`);\n if (!article.threadmark || !article.date || !article.author) {\n console.log(`Warning: Missing data in one or more fields for article ${index + 1}`);\n }\n });\n } else {\n console.log(`No articles detected.`);\n }\n\n return result;\n } catch (error) {\n console.log(`Error occurred while scraping URL: ${url}`);\n console.log(`Error details: ${error.message}`);\n console.log(`Error stack: ${error.stack}`);\n throw error;\n }\n}\n\nasync function getApiKey() {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n return apiKey;\n}\n\nasync function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/scrape\") {\n try {\n const { url } = await request.json();\n console.log(`Processing scrape request for URL: ${url}`);\n const result = await scrapePage(url);\n console.log(`Successfully scraped URL: ${url}`);\n return new Response(JSON.stringify(result), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.log(`Error occurred while processing request. Error details: ${error.message}`);\n console.log(`Error stack trace: ${error.stack}`);\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n }\n\n console.log(\"Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Web Scraper\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst css = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"] {\n width: 70%;\n padding: 10px;\n margin-right: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;\n\nexport default server;","deleted","label",[98],"permalink","https://www.val.town/x/willthereader/FanFicScraper/code/main.tsx","extension","actionData","errors"]