\\s*]*href=\"([^\"]*)\"[^>]*>([^<]*?):<\\/strong><\\/a>([^]*?)\\[(\\d+)\\]\\s*<\\/p>/g;\n\n let reviewMatch;\n while ((reviewMatch = reviewRegex.exec(postHtml)) !== null) {\n const reviewer_url = reviewMatch[1];\n const reviewer = reviewMatch[2].trim();\n let review_text = reviewMatch[3].trim();\n const review_score = parseInt(reviewMatch[4]);\n\n // Clean up the review text - remove HTML tags and extra whitespace\n review_text = review_text\n .replace(/<[^>]*>/g, \"\") // Remove HTML tags\n .replace(/\\s+/g, \" \") // Normalize whitespace\n .replace(/–/g, \"–\") // Decode HTML entities\n .replace(/&/g, \"&\")\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/"/g, \"\\\"\")\n .replace(/’/g, \"'\")\n .replace(/“/g, \"\\\"\")\n .replace(/”/g, \"\\\"\")\n .trim();\n\n // Only add if we have the essential data\n if (reviewer && review_text && review_score) {\n reviews.push({\n song_index: songIndex,\n artist,\n song_title,\n href,\n overall_score,\n reviewer,\n reviewer_url,\n review_text,\n review_score,\n });\n }\n }\n\n return reviews;\n}\n\n// Function to find all post divs in the HTML\nfunction findPostDivs(html: string): string[] {\n const postDivs: string[] = [];\n\n // Find all li elements with the post class\n const postRegex = /]*class=\"[^\"]*wp-block-post[^\"]*\"[^>]*>(.*?)<\\/li>/gs;\n\n let match;\n while ((match = postRegex.exec(html)) !== null) {\n postDivs.push(match[0]);\n }\n\n return postDivs;\n}\n\n// Main execution - process multiple pages\nconst pagesTotal = 5;\nlet songIndex = 0;\n\n// Parse each post div and extract reviews\nconst allReviews: Array<{\n song_index: number;\n artist: string;\n song_title: string;\n href: string;\n overall_score: number | null;\n reviewer: string;\n reviewer_url: string;\n review_text: string;\n review_score: number;\n}> = [];\n\nfor (let i = 2; i <= pagesTotal; i++) {\n console.log(`Processing page ${i}/${pagesTotal}...`);\n const res = await fetch(url(i));\n const text = await res.text();\n\n // Find all post divs\n const postDivs = findPostDivs(text);\n console.log(`Found ${postDivs.length} post divs on page ${i}`);\n\n postDivs.forEach((postHtml) => {\n console.log(`Processing song ${songIndex}...`);\n const reviews = parsePostDiv(postHtml, songIndex);\n allReviews.push(...reviews);\n\n console.log(reviews);\n songIndex++; // Increment for each song/post\n });\n}\n\nconsole.log(`Total reviews extracted: ${allReviews.length}`);\nconsole.log(`Total songs processed: ${songIndex}`);\n\n// Insert all reviews into SQLite\nfor (const review of allReviews) {\n await sqlite.execute(\n `\n INSERT INTO ${TABLE_NAME} \n (song_index, artist, song_title, href, overall_score, reviewer, reviewer_url, review_text, review_score)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n `,\n [\n review.song_index,\n review.artist,\n review.song_title,\n review.href,\n review.overall_score,\n review.reviewer,\n review.reviewer_url,\n review.review_text,\n review.review_score,\n ],\n );\n}\n\nconsole.log(`Successfully cached ${allReviews.length} reviews to SQLite table: ${TABLE_NAME}`);\nconsole.log(\"Script execution completed!\");","deleted","label",[99],"permalink","https://www.val.town/x/alexwein/tsjScraper/code/main.tsx","extension","actionData","errors"]