[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_159":160,"_161":162},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_19":20,"_17":18,"_21":122,"_23":123,"_124":125,"_126":127,"_27":128,"_25":129,"_130":131,"_132":133,"_134":51,"_135":136,"_137":51},"routes/_app.x.$handle.$name.branch.$branch.version.$version.code.$",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":26,"_27":28,"_29":30,"_31":-5,"_32":33,"_34":35,"_36":-5},"handle","willthereader","name","ThreadmarkLISTFetcher","branchName","main","version",156,"path","main.tsx","project",{"_37":81,"_13":14,"_39":82,"_49":-5,"_83":84,"_52":53,"_85":-5,"_86":-5,"_87":88,"_89":61,"_90":84,"_91":-5,"_92":-5,"_93":94,"_55":56,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":100,"_101":100,"_102":103,"_104":103,"_105":103,"_106":100,"_107":100,"_108":109,"_110":111,"_112":113,"_50":51},"tree",[76],"file",{"_37":59,"_13":20,"_60":61,"_62":-5,"_63":64,"_65":18,"_66":51,"_39":67,"_17":68,"_69":70,"_71":72,"_73":51,"_74":-5,"_19":75},"contents",[],"upstream",{"_37":38,"_39":40,"_41":42,"_13":16,"_43":44,"_17":35,"_45":-5,"_46":-5,"_47":48,"_49":-5,"_50":51,"_52":53,"_54":12,"_55":56,"_57":58},"pull","ahead",164,"behind",143,"envVars","id","9dbd5418-27b0-11f0-aecd-569c3dd06744","createdAt",["D",1746229996260],"updatedAt",["D",1727710402039],"projectId","9dbd1426-27b0-11f0-aecd-569c3dd06744","forkedBranchId","forkedBranchVersion","lockfileKey","1746250683190-esm.town_v_willthereader_FanFicScraper@143-main","deletedAt","isOwned",false,"privacy","public","userHandle","orgId","76badc02-5066-4776-b1f9-963b50ea816c","projectName","FanFicScraper","afc750a0-27b0-11f0-aecd-569c3dd06744","fileId","ac5c3c28-27b0-11f0-aecd-569c3dd06744","parentId","branchId","ac590c7e-27b0-11f0-aecd-569c3dd06744","branchVersion","latest",["D",1728053136032],155,"fileType","http","content","import { versionLogger } from \"https://esm.town/v/willthereader/versionLoggger\";\n\nfunction App() {\n let url = \"\";\n let chapterTitles = \"\";\n let result = null;\n let loading = false;\n\n const handleSubmit = async (e) => {\n e.preventDefault();\n loading = true;\n updateUI();\n console.log(`Fetching threadmarks for URL: ${url}`);\n try {\n result = await fetchThreadmarksFromServer(url, chapterTitles);\n } catch (error) {\n console.log(`Error occurred while fetching threadmarks: ${error.message}`);\n result = { error: error.message };\n }\n loading = false;\n updateUI();\n };\n\n const updateUI = () => {\n const root = document.getElementById(\"root\");\n root.innerHTML = `\n

Threadmark Fetcher and Comparator

\n \n ${\n result\n ? `\n

Result:

\n ${\n result.error\n ? `\n

Error: ${result.error}

\n ${\n result.stack\n ? `\n

Error Details

${result.stack}

\n `\n : \"\"\n }\n

\n `\n : `\n

Correctly Retrieved (In Order):

${title}

\n \n

Retrieved Out of Order:

${\n item.expectedIndex === -1\n ? `${item.title} (Unexpected, found at position: ${item.foundIndex + 1})`\n : `${item.title} (Expected position: ${item.expectedIndex + 1}, Found position: ${\n item.foundIndex + 1\n })`\n }

Summary:

Total expected: ${result.totalExpected}
Total retrieved: ${result.totalRetrieved}
Correctly retrieved (in order): ${result.correctOrder.length}
Retrieved out of order: ${result.outOfOrder.length - result.unexpectedCount}
Unexpected threadmarks: ${result.unexpectedCount}
Not retrieved: ${result.notRetrievedCount}

\n \n

Chapter Titles Not Retrieved:

${title}

\n \n

All Retrieved Threadmarks:

\n ${threadmark.title}\n

\n `\n }\n

\n `\n : \"\"\n }\n

\n `;\n\n document.getElementById(\"threadmarkForm\").addEventListener(\"submit\", handleSubmit);\n document.getElementById(\"urlInput\").addEventListener(\"input\", (e) => {\n url = e.target.value;\n });\n document.getElementById(\"chapterTitlesInput\").addEventListener(\"input\", (e) => {\n chapterTitles = e.target.value;\n });\n };\n\n return { updateUI };\n}\n\nasync function fetchThreadmarksFromServer(url, chapterTitles) {\n const response = await fetch(\"/api/threadmarks\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ url, chapterTitles }),\n });\n const data = await response.json();\n if (!response.ok) {\n throw new Error(data.error || `HTTP error! status: ${response.status}`);\n }\n return data;\n}\n\nasync function getThreadmarkUrls(baseUrl, apiKey) {\n versionLogger(import.meta.url);\n console.log(`[START] Threadmark URL fetch for base URL: ${baseUrl}`);\n const startTime = Date.now();\n\n const categories = [1]; // Only testing category 1 (threadmarks)\n let allThreadmarks = [];\n\n for (const category of categories) {\n try {\n const threadmarkUrl = `${baseUrl}/threadmarks?hide_wrapper=1&display=embedded&threadmark_category=${category}`;\n console.log(`[INFO] Constructing URL for category ${category}: ${threadmarkUrl}`);\n\n const jsScenario = {\n instructions: [\n { wait_for: \".structItem-cell--main[data-xf-click=\\\"threadmark-fetcher\\\"]\" },\n { click: \".structItem-cell--main[data-xf-click=\\\"threadmark-fetcher\\\"]\" },\n { wait: 2000 },\n {\n evaluate: `\nfunction clickNextThreadmarkButton() {\n const buttons = document.querySelectorAll('.structItem-cell--main[data-xf-click=\"threadmark-fetcher\"]');\n console.log(\\`Found \\${buttons.length} threadmark buttons\\`);\n if (buttons.length > 0) {\n console.log(\\`Clicking button: \\${buttons[0].getAttribute('data-fetchurl')}\\`);\n buttons[0].click();\n return true;\n }\n console.log('No more buttons to click');\n return false;\n}\n\nlet attempts = 0;\nconst maxAttempts = 20; // Increased from 10 to 20\n\nfunction attemptClick() {\n console.log(\\`Attempt \\${attempts + 1} started\\`);\n if (attempts < maxAttempts) {\n if (clickNextThreadmarkButton()) {\n attempts++;\n console.log(\\`Attempt \\${attempts}: Clicked a button, waiting 3 seconds...\\`);\n setTimeout(() => {\n console.log(\\`Total threadmarks after click: \\${document.querySelectorAll('.structItem-title.threadmark_depth0').length}\\`);\n attemptClick();\n }, 3000);\n } else {\n console.log('No more buttons to click, ending attempts');\n }\n } else {\n console.log(\\`Reached maximum attempts (\\${maxAttempts})\\`);\n }\n console.log(\\`Total threadmarks found: \\${document.querySelectorAll('.structItem-title.threadmark_depth0').length}\\`);\n}\n\nconsole.log('Starting to click threadmark buttons');\nattemptClick();\n\n// Verify selectors\nconsole.log('Verifying selectors:');\nconsole.log(\\`Threadmark buttons: \\${document.querySelectorAll('.structItem-cell--main[data-xf-click=\"threadmark-fetcher\"]').length}\\`);\nconsole.log(\\`Threadmark titles: \\${document.querySelectorAll('.structItem-title.threadmark_depth0').length}\\`);\nconsole.log(\\`Hidden threadmarks indicator: \\${document.querySelectorAll('.structItem--threadmark-filler').length}\\`);\n\n// Log page structure\nconsole.log('Page structure:');\nconsole.log(document.body.innerHTML);\n`,\n },\n ],\n };\n\n const scrapingBeeUrl = `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(threadmarkUrl)\n }&render_js=true&block_resources=false&wait_for=.structItem-cell--main[data-xf-click=\"threadmark-fetcher\"]&js_scenario=${\n encodeURIComponent(JSON.stringify(jsScenario))\n }`;\n console.log(`[INFO] Sending GET request to ScrapingBee API for category ${category}`);\n console.log(`[DEBUG] JS Scenario: ${JSON.stringify(jsScenario, null, 2)}`);\n\n const requestStartTime = Date.now();\n const response = await fetch(scrapingBeeUrl);\n console.log(\n `Response received. Status: ${response.status}, Headers: ${\n JSON.stringify(Object.fromEntries(response.headers))\n }`,\n );\n console.log(\n `[INFO] Received response for category ${category} in ${\n Date.now() - requestStartTime\n } ms. Status: ${response.status}`,\n );\n console.log(`[DEBUG] Response headers: ${JSON.stringify(Object.fromEntries(response.headers), null, 2)}`);\n const requestDuration = Date.now() - requestStartTime;\n console.log(\n `[INFO] Received response for category ${category} in ${requestDuration} ms. Status: ${response.status}`,\n );\n\n console.log(\n `[DEBUG] Response headers for category ${category}: ${\n JSON.stringify(Object.fromEntries(response.headers), null, 2)\n }`,\n );\n\n if (!response.ok) {\n const errorBody = await response.text();\n console.error(\n `[ERROR] Server error for category ${category}. Status: ${response.status}, Response body: ${errorBody}`,\n );\n throw new Error(`Server error: ${response.status} - ${errorBody}`);\n }\n\n const data = await response.text();\n console.log(\n `Raw data received. Length: ${data.length}, First 500 chars: ${data.slice(0, 500)}, Last 500 chars: ${\n data.slice(-500)\n }`,\n );\n console.log(`[INFO] Data received for category ${category}. Length: ${data.length} characters`);\n\n if (data.length === 0) {\n console.error(`[ERROR] Received empty response for category ${category}`);\n throw new Error(\"Received empty response from server\");\n }\n\n console.log(`[DEBUG] Raw data snippet for category ${category}: ${data.slice(0, 2000)}...`);\n console.log(`[DEBUG] Last part of raw data: ${data.slice(-2000)}`);\n console.log(`[DEBUG] Total data length: ${data.length}`);\n console.log(`[DEBUG] Checking for pagination: ${data.includes(\"pagination\") ? \"Found\" : \"Not found\"}`);\n\n const threadmarks = parseThreadmarks(data);\n console.log(\n `Parsed ${threadmarks.length} threadmarks. First: ${JSON.stringify(threadmarks[0])}, Last: ${\n JSON.stringify(threadmarks[threadmarks.length - 1])\n }`,\n );\n console.log(`[INFO] Parsed ${threadmarks.length} threadmarks from category ${category}`);\n if (threadmarks.length > 0) {\n console.log(`[DEBUG] First 3 threadmarks: ${JSON.stringify(threadmarks.slice(0, 3), null, 2)}`);\n } else {\n console.log(`[WARN] No threadmarks found in category ${category}`);\n }\n console.log(`[INFO] Parsed ${threadmarks.length} threadmarks from category ${category}`);\n\n if (threadmarks.length === 0) {\n console.warn(`[WARN] No threadmarks found in category ${category}`);\n } else {\n console.log(\n `[INFO] First 3 threadmarks from category ${category}: ${JSON.stringify(threadmarks.slice(0, 3), null, 2)}`,\n );\n }\n\n allThreadmarks = allThreadmarks.concat(threadmarks);\n } catch (error) {\n console.error(`[ERROR] Exception during fetching or parsing for category ${category}:`);\n console.error(`[ERROR] Error name: ${error.name}`);\n console.error(`[ERROR] Error message: ${error.message}`);\n console.error(`[DEBUG] Error stack: ${error.stack}`);\n throw error; // Re-throw the error to be caught by the caller\n }\n }\n\n const totalDuration = Date.now() - startTime;\n console.log(\n `[END] Finished fetching threadmarks for all categories in ${totalDuration} ms. Total threadmarks found: ${allThreadmarks.length}`,\n );\n\n if (allThreadmarks.length > 0) {\n console.log(`[INFO] First 3 threadmarks overall: ${JSON.stringify(allThreadmarks.slice(0, 3), null, 2)}`);\n return allThreadmarks;\n } else {\n console.warn(`[WARN] No threadmarks found across all categories`);\n throw new Error(\"No threadmarks found\");\n }\n}\n\nfunction parseThreadmarks(data) {\n console.log(`[INFO] Starting to parse threadmarks from HTML data. Total data length: ${data.length}`);\n const threadmarks = [];\n const regex = /

[\\s\\S]*?]*href=\"([^\"]*)\"[^>]*>([\\s\\S]*?)<\\/a>/g;\n let match;\n let count = 0;\n\n while ((match = regex.exec(data)) !== null) {\n count++;\n const url = match[1];\n const title = match[2].replace(/<[^>]*>/g, \"\").trim();\n threadmarks.push({ url, title });\n if (count % 100 === 0) {\n console.log(`[INFO] Processed ${count} threadmarks so far`);\n }\n }\n\n console.log(`[INFO] Extracted ${threadmarks.length} threadmarks from the HTML data`);\n console.log(`[DEBUG] First threadmark: ${JSON.stringify(threadmarks[0])}`);\n console.log(`[DEBUG] Last threadmark: ${JSON.stringify(threadmarks[threadmarks.length - 1])}`);\n return threadmarks;\n}\n\nfunction compareThreadmarks(expectedTitles, retrievedThreadmarks) {\n const expectedTitlesArray = expectedTitles.split(\"\\n\").map(title => title.trim()).filter(title => title !== \"\");\n const retrievedTitles = retrievedThreadmarks.map(threadmark => threadmark.title);\n\n const correctOrder = [];\n const outOfOrder = [];\n let notRetrievedCount = expectedTitlesArray.length;\n const notRetrieved = [...expectedTitlesArray];\n let lastCorrectIndex = -1;\n\n retrievedTitles.forEach((title, index) => {\n const expectedIndex = expectedTitlesArray.indexOf(title);\n if (expectedIndex !== -1) {\n notRetrievedCount--;\n if (expectedIndex > lastCorrectIndex) {\n correctOrder.push(title);\n lastCorrectIndex = expectedIndex;\n notRetrieved.splice(notRetrieved.indexOf(title), 1);\n } else {\n outOfOrder.push({ title, expectedIndex, foundIndex: index });\n }\n } else {\n outOfOrder.push({ title, expectedIndex: -1, foundIndex: index });\n }\n });\n\n return {\n correctOrder,\n outOfOrder,\n notRetrievedCount,\n allThreadmarks: retrievedThreadmarks,\n totalExpected: expectedTitlesArray.length,\n totalRetrieved: retrievedTitles.length,\n notRetrieved,\n unexpectedCount: retrievedTitles.length\n - (correctOrder.length + outOfOrder.length - (outOfOrder.filter(item => item.expectedIndex === -1).length)),\n };\n}\n\nfunction client() {\n console.log(\"Initializing client-side application\");\n const app = App();\n app.updateUI();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nexport default async function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/api/threadmarks\") {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.error(\"[ERROR] API key not found in environment variables\");\n return new Response(JSON.stringify({ error: \"API key not found in environment variables\" }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n\n try {\n const { url, chapterTitles } = await request.json();\n console.log(`[INFO] Received request to fetch threadmarks for URL: ${url}`);\n const threadmarks = await getThreadmarkUrls(url, apiKey);\n console.log(`[INFO] Retrieved ${threadmarks.length} threadmarks`);\n\n try {\n const comparisonResult = compareThreadmarks(chapterTitles, threadmarks);\n console.log(`[INFO] Comparison completed`);\n\n return new Response(JSON.stringify(comparisonResult), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (comparisonError) {\n console.error(\"[ERROR] Error in threadmark comparison:\");\n console.error(`[ERROR] Error name: ${comparisonError.name}`);\n console.error(`[ERROR] Error message: ${comparisonError.message}`);\n console.error(`[DEBUG] Error stack: ${comparisonError.stack}`);\n throw comparisonError; // Re-throw to be caught by the outer catch block\n }\n } catch (error) {\n console.error(\"[ERROR] Error in threadmark fetching or comparison:\");\n console.error(`[ERROR] Error name: ${error.name}`);\n console.error(`[ERROR] Error message: ${error.message}`);\n console.error(`[DEBUG] Error stack: ${error.stack}`);\n return new Response(\n JSON.stringify({\n error: `Error fetching threadmarks: ${error.message}`,\n stack: error.stack,\n }),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n }\n\n console.log(\"[INFO] Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Threadmark Fetcher and Comparator\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst CSS = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"], textarea {\n width: 100%;\n padding: 10px;\n margin-bottom: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;","deleted","label",[20],{"_37":59,"_13":20,"_60":61,"_62":-5,"_63":64,"_65":18,"_66":51,"_39":77,"_17":68,"_69":70,"_71":72,"_73":51,"_74":-5,"_78":79,"_19":80},["D",1728053136032],"children",[],[20],"ac58cac0-27b0-11f0-aecd-569c3dd06744",["D",1746230020769],"legacyEnvVars",true,"description","imageUrl","userId","e5615e9f-07dd-4d28-aaa1-9ee23a41135a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_121":118},"org",{"_11":12,"_117":118,"_87":88,"_119":120},"branch",{"_17":33,"_13":16,"_37":64,"_43":81,"_45":38,"_46":94,"_114":53,"_115":44,"_116":56,"_57":14,"_54":12},"forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJObXZ0bE1rbVBEMkhHNzR3SmVmMHc3eldBQiJ9","tier","pro","profileImageUrl",{"_37":81,"_13":14,"_39":155,"_49":-5,"_83":84,"_52":53,"_85":-5,"_86":-5,"_87":88,"_89":61,"_90":84,"_91":-5,"_92":-5,"_93":94,"_55":56,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":100,"_101":100,"_102":103,"_104":103,"_105":103,"_106":100,"_107":100,"_108":156,"_110":157,"_112":158,"_50":51},[152],"files",[152],"counts",{"_142":94,"_143":144,"_145":144,"_36":146,"_147":148},[],{"_37":59,"_13":20,"_60":61,"_62":-5,"_63":64,"_65":18,"_66":51,"_39":67,"_17":68,"_69":70,"_71":72,"_73":51,"_74":-5,"_19":141},"upstreamProject",{"_37":44,"_13":58,"_85":-5,"_86":-5,"_52":53,"_108":140},"upstreamBranch",{"_37":38,"_39":138,"_41":139,"_13":16,"_43":44,"_17":35,"_45":-5,"_46":-5,"_47":48,"_49":-5,"_50":51,"_52":53,"_54":12,"_55":56,"_57":58},"emptyEnvVars","flags",[],"navCollapsed",["D",1746229996260],["D",1727710402039],{"_11":12,"_121":118},[20],"pulls","branches",1,"forks",{"_149":94,"_151":94},"connections",{"_149":94,"_150":94},"count","unfilledCount","emptyCount",{"_37":59,"_13":20,"_60":61,"_62":-5,"_63":64,"_65":18,"_66":51,"_39":77,"_17":68,"_69":70,"_71":72,"_73":51,"_74":-5,"_78":153,"_19":154},[],[20],["D",1746230020769],{"_11":12,"_121":118},{"_11":12,"_117":118,"_87":88,"_119":120},{"_17":33,"_13":16,"_37":64,"_43":81,"_45":38,"_46":94,"_114":53,"_115":44,"_116":56,"_57":14,"_54":12},"ctx",{"_108":-5,"_192":193,"_194":195,"_196":51,"_197":51,"_198":51,"_199":-5,"_200":-7,"_201":-7,"_202":94,"_203":51,"_204":51,"_205":-5,"_206":51,"_207":51},"ENV",{"_163":164,"_165":166,"_167":168,"_169":16,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","f1316bb1ce3ad41ce227a17bcad7725ad044419f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","API_URL","https://api.val.town","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier"]