Threadmark Fetcher and Comparator

\n \n ${\n result\n ? `\n

Result:

\n ${\n result.error\n ? `\n

Error: ${result.error}

\n ${\n result.stack\n ? `\n

Error Details

${result.stack}

\n `\n : \"\"\n }\n

\n `\n : `\n

Correctly Retrieved (In Order):

${title}

\n \n

Retrieved Out of Order:

${\n item.expectedIndex === -1\n ? `${item.title} (Unexpected, found at position: ${item.foundIndex + 1})`\n : `${item.title} (Expected position: ${item.expectedIndex + 1}, Found position: ${\n item.foundIndex + 1\n })`\n }

Summary:

Total expected: ${result.totalExpected}
Total retrieved: ${result.totalRetrieved}
Correctly retrieved (in order): ${result.correctOrder.length}
Retrieved out of order: ${result.outOfOrder.length - result.unexpectedCount}
Unexpected threadmarks: ${result.unexpectedCount}
Not retrieved: ${result.notRetrievedCount}

\n \n

Chapter Titles Not Retrieved:

${title}

\n \n

All Retrieved Threadmarks:

\n ${threadmark.title}\n

\n `\n }\n

\n `\n : \"\"\n }\n

\n `;\n\n document.getElementById(\"threadmarkForm\").addEventListener(\"submit\", handleSubmit);\n document.getElementById(\"urlInput\").addEventListener(\"input\", (e) => {\n url = e.target.value;\n });\n document.getElementById(\"chapterTitlesInput\").addEventListener(\"input\", (e) => {\n chapterTitles = e.target.value;\n });\n };\n\n return { updateUI };\n}\n\nasync function fetchThreadmarksFromServer(url, chapterTitles) {\n const response = await fetch(\"/api/threadmarks\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ url, chapterTitles }),\n });\n const data = await response.json();\n if (!response.ok) {\n throw new Error(data.error || `HTTP error! status: ${response.status}`);\n }\n return data;\n}\n\nasync function getThreadmarkUrls(baseUrl, apiKey) {\n versionLogger(import.meta.url);\n console.log(`[START] Threadmark URL fetch for base URL: ${baseUrl}`);\n const startTime = Date.now();\n\n const categories = [1]; // Only testing category 1 (threadmarks)\n let allThreadmarks = [];\n\n for (const category of categories) {\n try {\n const threadmarkUrl = `${baseUrl}/threadmarks?hide_wrapper=1&display=embedded&threadmark_category=${category}`;\n console.log(`[INFO] Constructing URL for category ${category}: ${threadmarkUrl}`);\n\n const scrapingBeeUrl = `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(threadmarkUrl)\n }&render_js=true`;\n console.log(`[INFO] Sending GET request to ScrapingBee API for category ${category}`);\n\n const requestStartTime = Date.now();\n const response = await fetch(scrapingBeeUrl);\n console.log(\n `Response received. Status: ${response.status}, Headers: ${\n JSON.stringify(Object.fromEntries(response.headers))\n }`,\n );\n console.log(\n `[INFO] Received response for category ${category} in ${\n Date.now() - requestStartTime\n } ms. Status: ${response.status}`,\n );\n console.log(`[DEBUG] Response headers: ${JSON.stringify(Object.fromEntries(response.headers), null, 2)}`);\n const requestDuration = Date.now() - requestStartTime;\n console.log(\n `[INFO] Received response for category ${category} in ${requestDuration} ms. Status: ${response.status}`,\n );\n\n console.log(\n `[DEBUG] Response headers for category ${category}: ${\n JSON.stringify(Object.fromEntries(response.headers), null, 2)\n }`,\n );\n\n if (!response.ok) {\n const errorBody = await response.text();\n console.error(\n `[ERROR] Server error for category ${category}. Status: ${response.status}, Response body: ${errorBody}`,\n );\n throw new Error(`Server error: ${response.status} - ${errorBody}`);\n }\n\n const data = await response.text();\n console.log(\n `Raw data received. Length: ${data.length}, First 500 chars: ${data.slice(0, 500)}, Last 500 chars: ${\n data.slice(-500)\n }`,\n );\n console.log(`[INFO] Data received for category ${category}. Length: ${data.length} characters`);\n\n if (data.length === 0) {\n console.error(`[ERROR] Received empty response for category ${category}`);\n throw new Error(\"Received empty response from server\");\n }\n\n console.log(`[DEBUG] Raw data snippet for category ${category}: ${data.slice(0, 2000)}...`);\n console.log(`[DEBUG] Last part of raw data: ${data.slice(-2000)}`);\n console.log(`[DEBUG] Total data length: ${data.length}`);\n console.log(`[DEBUG] Checking for pagination: ${data.includes(\"pagination\") ? \"Found\" : \"Not found\"}`);\n\n const threadmarks = parseThreadmarks(data);\n console.log(\n `Parsed ${threadmarks.length} threadmarks. First: ${JSON.stringify(threadmarks[0])}, Last: ${\n JSON.stringify(threadmarks[threadmarks.length - 1])\n }`,\n );\n console.log(`[INFO] Parsed ${threadmarks.length} threadmarks from category ${category}`);\n if (threadmarks.length > 0) {\n console.log(`[DEBUG] First 3 threadmarks: ${JSON.stringify(threadmarks.slice(0, 3), null, 2)}`);\n } else {\n console.log(`[WARN] No threadmarks found in category ${category}`);\n }\n console.log(`[INFO] Parsed ${threadmarks.length} threadmarks from category ${category}`);\n\n if (threadmarks.length === 0) {\n console.warn(`[WARN] No threadmarks found in category ${category}`);\n } else {\n console.log(\n `[INFO] First 3 threadmarks from category ${category}: ${JSON.stringify(threadmarks.slice(0, 3), null, 2)}`,\n );\n }\n\n allThreadmarks = allThreadmarks.concat(threadmarks);\n } catch (error) {\n console.error(`[ERROR] Exception during fetching or parsing for category ${category}:`);\n console.error(`[ERROR] Error name: ${error.name}`);\n console.error(`[ERROR] Error message: ${error.message}`);\n console.error(`[DEBUG] Error stack: ${error.stack}`);\n throw error; // Re-throw the error to be caught by the caller\n }\n }\n\n const totalDuration = Date.now() - startTime;\n console.log(\n `[END] Finished fetching threadmarks for all categories in ${totalDuration} ms. Total threadmarks found: ${allThreadmarks.length}`,\n );\n\n if (allThreadmarks.length > 0) {\n console.log(`[INFO] First 3 threadmarks overall: ${JSON.stringify(allThreadmarks.slice(0, 3), null, 2)}`);\n return allThreadmarks;\n } else {\n console.warn(`[WARN] No threadmarks found across all categories`);\n throw new Error(\"No threadmarks found\");\n }\n}\n\nfunction parseThreadmarks(data) {\n console.log(`[INFO] Starting to parse threadmarks from HTML data. Total data length: ${data.length}`);\n const threadmarks = [];\n const regex = /

[\\s\\S]*?]*href=\"([^\"]*)\"[^>]*>([\\s\\S]*?)<\\/a>/g;\n let match;\n\n while ((match = regex.exec(data)) !== null) {\n const url = match[1];\n const title = match[2].replace(/<[^>]*>/g, \"\").trim();\n // console.log(`[DEBUG] Found threadmark - URL: ${url}, Title: ${title}`);\n threadmarks.push({ url, title });\n }\n\n console.log(`[INFO] Extracted ${threadmarks.length} threadmarks from the HTML data`);\n return threadmarks;\n}\n\nfunction compareThreadmarks(expectedTitles, retrievedThreadmarks) {\n const expectedTitlesArray = expectedTitles.split(\"\\n\").map(title => title.trim()).filter(title => title !== \"\");\n const retrievedTitles = retrievedThreadmarks.map(threadmark => threadmark.title);\n\n const correctOrder = [];\n const outOfOrder = [];\n let notRetrievedCount = expectedTitlesArray.length;\n const notRetrieved = [...expectedTitlesArray];\n let lastCorrectIndex = -1;\n\n retrievedTitles.forEach((title, index) => {\n const expectedIndex = expectedTitlesArray.indexOf(title);\n if (expectedIndex !== -1) {\n notRetrievedCount--;\n if (expectedIndex > lastCorrectIndex) {\n correctOrder.push(title);\n lastCorrectIndex = expectedIndex;\n notRetrieved.splice(notRetrieved.indexOf(title), 1);\n } else {\n outOfOrder.push({ title, expectedIndex, foundIndex: index });\n }\n } else {\n outOfOrder.push({ title, expectedIndex: -1, foundIndex: index });\n }\n });\n\n return {\n correctOrder,\n outOfOrder,\n notRetrievedCount,\n allThreadmarks: retrievedThreadmarks,\n totalExpected: expectedTitlesArray.length,\n totalRetrieved: retrievedTitles.length,\n notRetrieved,\n unexpectedCount: retrievedTitles.length\n - (correctOrder.length + outOfOrder.length - (outOfOrder.filter(item => item.expectedIndex === -1).length)),\n };\n}\n\nfunction client() {\n console.log(\"Initializing client-side application\");\n const app = App();\n app.updateUI();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nexport default async function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/api/threadmarks\") {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.error(\"[ERROR] API key not found in environment variables\");\n return new Response(JSON.stringify({ error: \"API key not found in environment variables\" }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n\n try {\n const { url, chapterTitles } = await request.json();\n console.log(`[INFO] Received request to fetch threadmarks for URL: ${url}`);\n const threadmarks = await getThreadmarkUrls(url, apiKey);\n console.log(`[INFO] Retrieved ${threadmarks.length} threadmarks`);\n\n try {\n const comparisonResult = compareThreadmarks(chapterTitles, threadmarks);\n console.log(`[INFO] Comparison completed`);\n\n return new Response(JSON.stringify(comparisonResult), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (comparisonError) {\n console.error(\"[ERROR] Error in threadmark comparison:\");\n console.error(`[ERROR] Error name: ${comparisonError.name}`);\n console.error(`[ERROR] Error message: ${comparisonError.message}`);\n console.error(`[DEBUG] Error stack: ${comparisonError.stack}`);\n throw comparisonError; // Re-throw to be caught by the outer catch block\n }\n } catch (error) {\n console.error(\"[ERROR] Error in threadmark fetching or comparison:\");\n console.error(`[ERROR] Error name: ${error.name}`);\n console.error(`[ERROR] Error message: ${error.message}`);\n console.error(`[DEBUG] Error stack: ${error.stack}`);\n return new Response(\n JSON.stringify({\n error: `Error fetching threadmarks: ${error.message}`,\n stack: error.stack,\n }),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n }\n\n console.log(\"[INFO] Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Threadmark Fetcher and Comparator\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst CSS = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"], textarea {\n width: 100%;\n padding: 10px;\n margin-bottom: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;","deleted","label",[98],"permalink","https://www.val.town/x/willthereader/ThreadmarkLISTFetcher/code/main.tsx","extension","actionData","errors"]