import { versionLogger } from "https://esm.town/v/willthereader/versionLoggger"; function App() { let url = ""; let chapterTitles = ""; let result = null; let loading = false; const handleSubmit = async (e) => { e.preventDefault(); loading = true; updateUI(); console.log(`Fetching threadmarks for URL: ${url}`); try { result = await fetchThreadmarksFromServer(url, chapterTitles); } catch (error) { console.log(`Error occurred while fetching threadmarks: ${error.message}`); result = { error: error.message }; } loading = false; updateUI(); }; const updateUI = () => { const root = document.getElementById("root"); root.innerHTML = `

Threadmark Fetcher and Comparator

${ result ? `

Result:

${ result.error ? `

Error: ${result.error}

${ result.stack ? `

Error Details

${result.stack}

` : "" }

` : `

Correctly Retrieved (In Order):

${title}

Retrieved Out of Order:

${ item.expectedIndex === -1 ? `${item.title} (Unexpected, found at position: ${item.foundIndex + 1})` : `${item.title} (Expected position: ${item.expectedIndex + 1}, Found position: ${ item.foundIndex + 1 })` }

Summary:

Total expected: ${result.totalExpected}
Total retrieved: ${result.totalRetrieved}
Correctly retrieved (in order): ${result.correctOrder.length}
Retrieved out of order: ${result.outOfOrder.length - result.unexpectedCount}
Unexpected threadmarks: ${result.unexpectedCount}
Not retrieved: ${result.notRetrievedCount}

Chapter Titles Not Retrieved:

${title}

All Retrieved Threadmarks:

${threadmark.title}

` }

` : "" }

`; document.getElementById("threadmarkForm").addEventListener("submit", handleSubmit); document.getElementById("urlInput").addEventListener("input", (e) => { url = e.target.value; }); document.getElementById("chapterTitlesInput").addEventListener("input", (e) => { chapterTitles = e.target.value; }); }; return { updateUI }; } async function fetchThreadmarksFromServer(url, chapterTitles) { const response = await fetch("/api/threadmarks", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ url, chapterTitles }), }); const data = await response.json(); if (!response.ok) { throw new Error(data.error || `HTTP error! status: ${response.status}`); } return data; } async function getThreadmarkUrls(baseUrl, apiKey) { versionLogger(import.meta.url); console.log(`[START] Threadmark URL fetch for base URL: ${baseUrl}`); const startTime = Date.now(); const categories = [1]; // Only testing category 1 (threadmarks) let allThreadmarks = []; for (const category of categories) { try { const threadmarkUrl = `${baseUrl}/threadmarks?hide_wrapper=1&display=embedded&threadmark_category=${category}`; console.log(`[INFO] Constructing URL for category ${category}: ${threadmarkUrl}`); const scrapingBeeUrl = `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${ encodeURIComponent(threadmarkUrl) }&render_js=true`; console.log(`[INFO] Sending GET request to ScrapingBee API for category ${category}`); const requestStartTime = Date.now(); const response = await fetch(scrapingBeeUrl); console.log( `Response received. Status: ${response.status}, Headers: ${ JSON.stringify(Object.fromEntries(response.headers)) }`, ); console.log( `[INFO] Received response for category ${category} in ${ Date.now() - requestStartTime } ms. Status: ${response.status}`, ); console.log(`[DEBUG] Response headers: ${JSON.stringify(Object.fromEntries(response.headers), null, 2)}`); const requestDuration = Date.now() - requestStartTime; console.log( `[INFO] Received response for category ${category} in ${requestDuration} ms. Status: ${response.status}`, ); console.log( `[DEBUG] Response headers for category ${category}: ${ JSON.stringify(Object.fromEntries(response.headers), null, 2) }`, ); if (!response.ok) { const errorBody = await response.text(); console.error( `[ERROR] Server error for category ${category}. Status: ${response.status}, Response body: ${errorBody}`, ); throw new Error(`Server error: ${response.status} - ${errorBody}`); } const data = await response.text(); console.log( `Raw data received. Length: ${data.length}, First 500 chars: ${data.slice(0, 500)}, Last 500 chars: ${ data.slice(-500) }`, ); console.log(`[INFO] Data received for category ${category}. Length: ${data.length} characters`); if (data.length === 0) { console.error(`[ERROR] Received empty response for category ${category}`); throw new Error("Received empty response from server"); } console.log(`[DEBUG] Raw data snippet for category ${category}: ${data.slice(0, 2000)}...`); console.log(`[DEBUG] Last part of raw data: ${data.slice(-2000)}`); console.log(`[DEBUG] Total data length: ${data.length}`); console.log(`[DEBUG] Checking for pagination: ${data.includes("pagination") ? "Found" : "Not found"}`); const threadmarks = parseThreadmarks(data); console.log( `Parsed ${threadmarks.length} threadmarks. First: ${JSON.stringify(threadmarks[0])}, Last: ${ JSON.stringify(threadmarks[threadmarks.length - 1]) }`, ); console.log(`[INFO] Parsed ${threadmarks.length} threadmarks from category ${category}`); if (threadmarks.length > 0) { console.log(`[DEBUG] First 3 threadmarks: ${JSON.stringify(threadmarks.slice(0, 3), null, 2)}`); } else { console.log(`[WARN] No threadmarks found in category ${category}`); } console.log(`[INFO] Parsed ${threadmarks.length} threadmarks from category ${category}`); if (threadmarks.length === 0) { console.warn(`[WARN] No threadmarks found in category ${category}`); } else { console.log( `[INFO] First 3 threadmarks from category ${category}: ${JSON.stringify(threadmarks.slice(0, 3), null, 2)}`, ); } allThreadmarks = allThreadmarks.concat(threadmarks); } catch (error) { console.error(`[ERROR] Exception during fetching or parsing for category ${category}:`); console.error(`[ERROR] Error name: ${error.name}`); console.error(`[ERROR] Error message: ${error.message}`); console.error(`[DEBUG] Error stack: ${error.stack}`); throw error; // Re-throw the error to be caught by the caller } } const totalDuration = Date.now() - startTime; console.log( `[END] Finished fetching threadmarks for all categories in ${totalDuration} ms. Total threadmarks found: ${allThreadmarks.length}`, ); if (allThreadmarks.length > 0) { console.log(`[INFO] First 3 threadmarks overall: ${JSON.stringify(allThreadmarks.slice(0, 3), null, 2)}`); return allThreadmarks; } else { console.warn(`[WARN] No threadmarks found across all categories`); throw new Error("No threadmarks found"); } } function parseThreadmarks(data) { console.log(`[INFO] Starting to parse threadmarks from HTML data. Total data length: ${data.length}`); const threadmarks = []; const regex = /

[\s\S]*?]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g; let match; while ((match = regex.exec(data)) !== null) { const url = match[1]; const title = match[2].replace(/<[^>]*>/g, "").trim(); // console.log(`[DEBUG] Found threadmark - URL: ${url}, Title: ${title}`); threadmarks.push({ url, title }); } console.log(`[INFO] Extracted ${threadmarks.length} threadmarks from the HTML data`); return threadmarks; } function compareThreadmarks(expectedTitles, retrievedThreadmarks) { const expectedTitlesArray = expectedTitles.split("\n").map(title => title.trim()).filter(title => title !== ""); const retrievedTitles = retrievedThreadmarks.map(threadmark => threadmark.title); const correctOrder = []; const outOfOrder = []; let notRetrievedCount = expectedTitlesArray.length; const notRetrieved = [...expectedTitlesArray]; let lastCorrectIndex = -1; retrievedTitles.forEach((title, index) => { const expectedIndex = expectedTitlesArray.indexOf(title); if (expectedIndex !== -1) { notRetrievedCount--; if (expectedIndex > lastCorrectIndex) { correctOrder.push(title); lastCorrectIndex = expectedIndex; notRetrieved.splice(notRetrieved.indexOf(title), 1); } else { outOfOrder.push({ title, expectedIndex, foundIndex: index }); } } else { outOfOrder.push({ title, expectedIndex: -1, foundIndex: index }); } }); return { correctOrder, outOfOrder, notRetrievedCount, allThreadmarks: retrievedThreadmarks, totalExpected: expectedTitlesArray.length, totalRetrieved: retrievedTitles.length, notRetrieved, unexpectedCount: retrievedTitles.length - (correctOrder.length + outOfOrder.length - (outOfOrder.filter(item => item.expectedIndex === -1).length)), }; } function client() { console.log("Initializing client-side application"); const app = App(); app.updateUI(); } if (typeof document !== "undefined") { client(); } export default async function server(request) { console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`); if (request.method === "POST" && new URL(request.url).pathname === "/api/threadmarks") { const apiKey = Deno.env.get("ScrapingBeeAPIkey"); if (!apiKey) { console.error("[ERROR] API key not found in environment variables"); return new Response(JSON.stringify({ error: "API key not found in environment variables" }), { status: 500, headers: { "Content-Type": "application/json" }, }); } try { const { url, chapterTitles } = await request.json(); console.log(`[INFO] Received request to fetch threadmarks for URL: ${url}`); const threadmarks = await getThreadmarkUrls(url, apiKey); console.log(`[INFO] Retrieved ${threadmarks.length} threadmarks`); try { const comparisonResult = compareThreadmarks(chapterTitles, threadmarks); console.log(`[INFO] Comparison completed`); return new Response(JSON.stringify(comparisonResult), { headers: { "Content-Type": "application/json" }, }); } catch (comparisonError) { console.error("[ERROR] Error in threadmark comparison:"); console.error(`[ERROR] Error name: ${comparisonError.name}`); console.error(`[ERROR] Error message: ${comparisonError.message}`); console.error(`[DEBUG] Error stack: ${comparisonError.stack}`); throw comparisonError; // Re-throw to be caught by the outer catch block } } catch (error) { console.error("[ERROR] Error in threadmark fetching or comparison:"); console.error(`[ERROR] Error name: ${error.name}`); console.error(`[ERROR] Error message: ${error.message}`); console.error(`[DEBUG] Error stack: ${error.stack}`); return new Response( JSON.stringify({ error: `Error fetching threadmarks: ${error.message}`, stack: error.stack, }), { status: 500, headers: { "Content-Type": "application/json" }, }, ); } } console.log("[INFO] Serving HTML page"); return new Response( ` Threadmark Fetcher and Comparator

`, { headers: { "Content-Type": "text/html" }, }, ); } const CSS = ` body { font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; } input[type="url"], textarea { width: 100%; padding: 10px; margin-bottom: 10px; } button { padding: 10px 20px; } pre { background-color: #f4f4f4; padding: 10px; border-radius: 5px; white-space: pre-wrap; word-wrap: break-word; } `;

ThreadmarkLISTFetcher