FanFic Scraper

\n setUrl(e.target.value)}\n placeholder=\"Enter URL to scrape\"\n required\n />\n \n \n {result && (\n

Result:

{result.title}

Content:

Chapters:

\n
Threadmark: {article.threadmark}
\n
Date: {article.date}
\n
Author: {article.author}
\n
\n

\n\n {result.error &&

Error: {result.error}

}\n

\n )}\n

\n );\n}\n\nfunction client() {\n console.log(\"Initializing client-side React application\");\n createRoot(document.getElementById(\"root\")).render();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nconst _REGEX_THREAD_URL = /threads\\/(?\\d+)/;\nconst _REGEX_POST_URL = /posts\\/(?\\d+)/;\n\nfunction normalizeUrl(url) {\n const threadMatch = url.match(_REGEX_THREAD_URL);\n const postMatch = url.match(_REGEX_POST_URL);\n\n if (postMatch) {\n return `https://forums.spacebattles.com/posts/${postMatch.groups.id}/`;\n } else if (threadMatch) {\n return `https://forums.spacebattles.com/threads/${threadMatch.groups.id}/`;\n }\n return url;\n}\n\nasync function scrapePage(url) {\n console.log(`Starting to scrape page: ${url}`);\n const apiKey = await getApiKey();\n\n url = normalizeUrl(url);\n console.log(`Normalized URL for scraping: ${url}`);\n\n const extractRules = {\n title: \"h1.p-title-value\",\n prefixTags: \"div.p-body-header a[href*='prefix_id']\",\n posts: {\n selector: \"article.message--post\",\n type: \"list\",\n output: {\n id: \"@data-content\",\n content: \"article.message-body div.bbWrapper\",\n author: \"div.message-userDetails .username\",\n date: \"div.message-attribution-main time\",\n lastEdit: \"div.message-lastEdit time\",\n isSticky: \".sticky-container\",\n },\n },\n threadmarksTop: \"div.block-outer-recent-threadmarks\",\n threadmarks: {\n selector: \"div.block-outer-recent-threadmarks a.menuTrigger[href*='threadmarks']\",\n type: \"list\",\n output: {\n category: \"@href\",\n name: \"text\",\n },\n },\n };\n\n const result = await scrapePageWithRules(url, apiKey, extractRules);\n return processResults(result);\n}\n\nfunction processThreadmarks(threadmarks) {\n return threadmarks.map(tm => {\n let category = \"1\";\n const categoryMatch = tm.category.match(/threadmark_category=(\\d+)/);\n if (categoryMatch) {\n category = categoryMatch[1];\n }\n return {\n category: category,\n name: tm.name,\n };\n });\n}\n\nclass NetworkError extends Error {\n constructor(message) {\n super(message);\n this.name = \"NetworkError\";\n }\n}\n\nclass APIError extends Error {\n constructor(message, status) {\n super(message);\n this.name = \"APIError\";\n this.status = status;\n }\n}\n\nclass ParsingError extends Error {\n constructor(message) {\n super(message);\n this.name = \"ParsingError\";\n }\n}\n\nasync function scrapePageWithRules(url, apiKey, extractRules) {\n console.log(`Sending request to ScrapingBee API for URL: ${url}`);\n try {\n const response = await fetch(\n `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(url)\n }&render_js=true&json_response=true&extract_rules=${encodeURIComponent(JSON.stringify(extractRules))}`,\n );\n\n if (!response.ok) {\n throw new APIError(`HTTP error! status: ${response.status}`, response.status);\n }\n\n console.log(\"Response received from ScrapingBee API\");\n console.log(`Cost: ${response.headers.get(\"Spb-cost\")}`);\n console.log(`Initial status code: ${response.headers.get(\"Spb-initial-status-code\")}`);\n console.log(`Resolved URL: ${response.headers.get(\"Spb-resolved-url\")}`);\n\n let result;\n try {\n result = await response.json();\n } catch (jsonError) {\n throw new ParsingError(\"Failed to parse JSON response\");\n }\n\n console.log(\"ScrapingBee API response parsed successfully\");\n return result;\n } catch (error) {\n if (error instanceof TypeError) {\n console.error(\"Network error:\", error.message);\n throw new NetworkError(\"Failed to connect to the API\");\n } else if (error instanceof APIError) {\n console.error(\"API error:\", error.message, \"Status:\", error.status);\n throw error;\n } else if (error instanceof ParsingError) {\n console.error(\"Parsing error:\", error.message);\n throw error;\n } else {\n console.error(\"Unexpected error:\", error.message);\n throw new Error(\"An unexpected error occurred\");\n }\n }\n}\n\nfunction processResults(results) {\n console.log(\"Processing results:\", JSON.stringify(results, null, 2));\n if (!results || results.length === 0) {\n console.log(\"No results to process\");\n return {\n title: \"No title found\",\n prefixTags: [],\n firstPost: null,\n threadmarks: [],\n posts: [],\n };\n }\n\n const processedResult = {\n title: results[0].title || \"No title found\",\n prefixTags: results[0].prefixTags || [],\n firstPost: null,\n threadmarks: results[0].threadmarks ? processThreadmarks(results[0].threadmarks) : [],\n posts: [],\n };\n\n if (results[0].posts && Array.isArray(results[0].posts)) {\n processedResult.posts = results[0].posts.map(processPost);\n processedResult.firstPost = processedResult.posts.find(post => !post.isSticky) || processedResult.posts[0];\n }\n\n console.log(`Final processed result: ${JSON.stringify(processedResult, null, 2)}`);\n return processedResult;\n}\n\nfunction processPost(post) {\n return {\n id: post.id,\n content: post.content,\n author: post.author,\n date: new Date(post.date),\n lastEdit: post.lastEdit ? new Date(post.lastEdit) : null,\n isSticky: !!post.isSticky,\n };\n}\n\nasync function getApiKey() {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n return apiKey;\n}\n\nexport default async function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/scrape\") {\n try {\n const { url } = await request.json();\n const result = await scrapePage(url);\n console.log(`Successfully scraped URL: ${url}`);\n console.log(`Scraping completed for URL: ${url}\n Result summary:\n Title: ${result.title}\n Number of posts: ${result.posts.length}`);\n return new Response(JSON.stringify(result), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.log(`Error occurred while processing request. Error details: ${error.message}`);\n console.log(`Error stack trace: ${error.stack}`);\n console.error(`Error occurred during scraping operation:\n Error message: ${error.message}\n Stack trace:\n ${error.stack}`);\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n }\n\n console.log(\"Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Web Scraper\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst css = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"] {\n width: 70%;\n padding: 10px;\n margin-right: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;","deleted","label","children",[],[154],"files",[152],"counts",{"_175":32,"_176":177,"_178":32,"_179":180},"pulls","branches",1,"forks","envVars",{"_181":32,"_182":32},"count","emptyCount","contents",[185],{"_102":153,"_95":154,"_155":115,"_156":-5,"_157":138,"_158":137,"_159":108,"_104":186,"_99":161,"_162":163,"_164":165,"_166":22,"_167":-5,"_98":187},["D",1726939990306],[154],"file","upstreamProject",{"_102":145,"_95":191,"_111":-5,"_112":-5,"_109":110,"_23":192},"FanFicScraper",{"_93":94,"_130":131},"upstreamBranch",{"_102":141,"_104":195,"_196":197,"_95":43,"_139":145,"_99":198,"_140":-5,"_142":-5,"_199":200,"_106":-5,"_149":22,"_109":110,"_148":94,"_16":119,"_147":191},["D",1746229996260],"updatedAt",["D",1727710402039],143,"lockfileKey","1746250683190-esm.town_v_willthereader_FanFicScraper@143-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.environment-variables",{"_93":94,"_95":96,"_100":207,"_179":212},{"_102":103,"_95":96,"_104":208,"_106":-5,"_107":108,"_109":110,"_111":-5,"_112":-5,"_14":113,"_114":115,"_116":108,"_117":-5,"_118":32,"_16":119,"_120":121,"_122":121,"_123":124,"_125":124,"_126":124,"_127":121,"_128":121,"_23":209,"_132":210,"_135":211,"_149":22},["D",1746229953451],{"_93":94,"_130":131},{"_93":94,"_134":131},{"_99":137,"_95":43,"_102":138,"_139":103,"_140":141,"_142":32,"_143":110,"_144":145,"_146":119,"_147":96,"_148":94},[],"actionData","errors"]

willthereader

v2FanFicScraper

Environment variables