[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_164":165,"_166":167},"routes/_app.x.$handle.$name",{"_95":94,"_32":45,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":57,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":31,"_56":57,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",81,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","6bcab73c-2794-11f0-9d52-569c3dd06744","name","fileType","http","content","import { Hono } from \"https://esm.sh/hono@3.10.2\";\nimport { encode } from \"https://esm.sh/js-base64@3.7.5\";\nimport { parse } from \"https://esm.sh/node-html-parser@6.1.5\";\n\nconst app = new Hono();\n\n// Expanded stopwords list with additional web-specific terms\nconst stopwords = new Set([\n 'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'because',\n 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'could', 'did', 'do', 'does', 'doing', 'down',\n 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'has', 'have', 'having', 'he', 'her', 'here', 'hers',\n 'herself', 'him', 'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'its', 'itself', 'just', 'me',\n 'more', 'most', 'my', 'myself', 'no', 'nor', 'not', 'now', 'of', 'off', 'on', 'once', 'only', 'or', 'other',\n 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some', 'such', 'than',\n 'that', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'these', 'they', 'this', 'those', 'through',\n 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which', 'while', 'who',\n 'whom', 'why', 'with', 'would', 'you', 'your', 'yours', 'yourself', 'yourselves',\n // Additional navigation-related terms\n 'back', 'contents', 'menu', 'home', 'next', 'previous', 'top', 'bottom', 'navigation', 'nav', 'sidebar', 'footer',\n 'header', 'main', 'article', 'section', 'page', 'skip', 'jump', 'return',\n // Web-specific terms\n 'image', 'photo', 'photograph', 'picture', 'fullscreen', 'share', 'reuse', 'features', 'topics', 'observer',\n 'getty', 'rex', 'shutterstock', 'ap', 'afp', 'reuters', 'epa', 'pa', 'alamy', 'getty images', 'associated press',\n 'agence france-presse', 'press association', 'copyright', 'rights reserved', 'all rights reserved',\n 'click', 'tap', 'swipe', 'scroll', 'zoom', 'enlarge', 'expand', 'collapse', 'hide', 'show',\n 'login', 'sign in', 'register', 'subscribe', 'newsletter', 'email', 'follow', 'like', 'comment', 'share',\n 'twitter', 'facebook', 'instagram', 'linkedin', 'pinterest', 'youtube', 'google',\n 'advertisement', 'sponsored', 'promoted', 'partner content', 'paid content',\n 'cookie', 'privacy', 'terms', 'conditions', 'policy', 'agreement',\n 'mobile', 'desktop', 'tablet', 'app', 'website', 'browser',\n 'search', 'find', 'results', 'query', 'keyword',\n 'loading', 'please wait', 'error', 'not found', '404', 'server error', '500',\n 'menu', 'navigation', 'sitemap', 'archive', 'category', 'tag', 'related', 'popular', 'trending',\n 'breaking news', 'latest', 'update', 'live', 'developing story',\n 'read more', 'continue reading', 'full article', 'paywall', 'subscribe now',\n 'comments', 'discussion', 'forum', 'community', 'join the conversation',\n 'share your thoughts', 'leave a comment', 'what do you think',\n 'fact check', 'analysis', 'opinion', 'editorial', 'column', 'blog post',\n 'video', 'audio', 'podcast', 'gallery', 'slideshow', 'interactive',\n 'download', 'upload', 'attachment', 'file', 'document',\n 'print', 'email', 'save', 'bookmark', 'favorite',\n 'font size', 'text size', 'dark mode', 'light mode', 'accessibility',\n 'support', 'help', 'faq', 'contact us', 'about us', 'team', 'careers',\n 'advertise', 'partnerships', 'sponsorships', 'media kit',\n 'rss feed', 'xml', 'api', 'widget', 'embed',\n 'copyright', 'trademark', 'legal', 'disclaimer', 'terms of use', 'privacy policy'\n]);\n\n// Expanded Synonym dictionary (unchanged)\nconst synonyms = {\n \"important\": [\"significant\", \"crucial\", \"vital\", \"essential\", \"key\"],\n // ... (rest of the synonyms dictionary remains unchanged)\n};\n\n// Simple sentiment analysis function (unchanged)\nfunction analyseSentiment(text) {\n const positiveWords = new Set([\n 'good', 'great', 'excellent', 'wonderful', 'amazing', 'fantastic', 'awesome',\n 'happy', 'joy', 'love', 'like', 'beautiful', 'nice', 'pleasant', 'delightful'\n ]);\n const negativeWords = new Set([\n 'bad', 'terrible', 'awful', 'horrible', 'dreadful', 'poor', 'disappointing',\n 'sad', 'unhappy', 'hate', 'dislike', 'ugly', 'unpleasant', 'nasty', 'cruel'\n ]);\n\n const words = text.toLowerCase().match(/\\b\\w+\\b/g) || [];\n let score = 0;\n words.forEach(word => {\n if (positiveWords.has(word)) score++;\n if (negativeWords.has(word)) score--;\n });\n\n let sentiment;\n if (score > 0) {\n sentiment = \"positive\";\n } else if (score < 0) {\n sentiment = \"negative\";\n } else {\n sentiment = \"neutral\";\n }\n\n return { score, sentiment };\n}\n\n// Improved TF-IDF implementation with additional filtering\nfunction calculateTFIDF(text, outputSize = 7) {\n const words = text.toLowerCase().match(/\\b[a-z]{2,}\\b/g) || [];\n const wordCount = words.length;\n const termFrequency = {};\n const documentFrequency = {};\n\n // Calculate term frequency and document frequency\n for (let i = 0; i < words.length; i++) {\n for (let j = 1; j <= 3 && i + j <= words.length; j++) {\n const phrase = words.slice(i, i + j).join(' ');\n if (!phrase.split(' ').every(word => stopwords.has(word))) {\n termFrequency[phrase] = (termFrequency[phrase] || 0) + 1;\n documentFrequency[phrase] = 1;\n }\n }\n }\n\n // Calculate TF-IDF with improved scoring\n const tfidf = {};\n Object.keys(termFrequency).forEach(term => {\n const tf = termFrequency[term] / wordCount;\n const idf = Math.log(wordCount / (documentFrequency[term] || 1)) + 1;\n const termWordCount = term.split(' ').length;\n // Boost score for multi-word phrases, less common terms, and longer phrases\n tfidf[term] = tf * idf * (termWordCount * 0.5) * (1 / Math.sqrt(termFrequency[term])) * (term.length / 10);\n });\n\n // Sort by score (descending) and get top unique phrases\n const uniquePhrases = new Set();\n return Object.entries(tfidf)\n .sort((a, b) => b[1] - a[1])\n .filter(([phrase, _]) => {\n if (uniquePhrases.size >= outputSize) return false;\n const words = phrase.split(' ');\n if (words.length > 3) return false;\n if (words.some(word => word.length < 3)) return false;\n if (words.every(word => stopwords.has(word))) return false;\n // Check if the phrase is a subset of an already included phrase\n if (Array.from(uniquePhrases).some(existingPhrase => existingPhrase.includes(phrase))) return false;\n // Additional filtering to remove unwanted terms\n if (phrase.includes('http') || phrase.includes('www') || phrase.includes('com')) return false;\n if (/\\d/.test(phrase)) return false; // Exclude phrases containing numbers\n if (phrase.length < 4) return false; // Exclude very short phrases\n uniquePhrases.add(phrase);\n return true;\n })\n .slice(0, outputSize);\n}\n\n// Updated function to calculate word frequency\nfunction calculateWordFrequency(text, topN = 30) {\n const words = text.toLowerCase().match(/\\b[a-z]{3,}\\b/g) || [];\n const frequency = {};\n words.forEach(word => {\n if (!stopwords.has(word) && !word.includes('http') && !word.includes('www') && !word.includes('com')) {\n frequency[word] = (frequency[word] || 0) + 1;\n }\n });\n return Object.entries(frequency)\n .sort((a, b) => b[1] - a[1])\n .slice(0, topN);\n}\n\n// Updated function to generate a fun fact\nfunction generateFunFact(text) {\n const wordCount = text.trim().split(/\\s+/).length;\n const charCount = text.length;\n const sentences = text.split(/[.!?]+/).length;\n const avgWordLength = wordCount > 0 ? charCount / wordCount : 0;\n\n const words = text.match(/\\b\\w+\\b/g) || [];\n const longestWord = words.length > 0 ? words.reduce((a, b) => a.length > b.length ? a : b) : 'N/A';\n\n const facts = [\n `Your text has an average word length of ${avgWordLength.toFixed(2)} characters. Shakespeare's average was about 3.99!`,\n `You've written ${sentences} sentence${sentences !== 1 ? 's' : ''}. A typical novel has about 50-60 sentences per page.`,\n `Your text is ${wordCount} words long. \"War and Peace\" by Leo Tolstoy is about 587,287 words!`,\n `Your longest word is \"${longestWord}\". The longest word in English is 189,819 letters long!`,\n `You've used ${charCount} characters. That's about ${(charCount / 280).toFixed(2)} tweets' worth of content!`\n ];\n\n return facts[Math.floor(Math.random() * facts.length)];\n}\n\n// Function to generate a simple summary (unchanged)\nfunction generateSummary(text, sentenceCount = 3) {\n const sentences = text.match(/[^\\.!\\?]+[\\.!\\?]+/g) || [];\n return sentences.slice(0, sentenceCount).join(' ');\n}\n\n// Updated function to clean the extracted text\nfunction cleanText(text) {\n return text\n .replace(/<[^>]*>/g, '') // Remove HTML tags\n .replace(/\\s+/g, ' ') // Replace multiple spaces with a single space\n .replace(/[^\\w\\s.,!?-]/g, ' ') // Replace special characters with spaces, except for some punctuation\n .replace(/\\b(https?:\\/\\/)?[^\\s]+\\.(com|org|net|edu|gov)\\b/g, '') // Remove URLs\n .replace(/\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b/g, '') // Remove email addresses\n .replace(/\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\b/g, '') // Remove IP addresses\n .replace(/\\b(back to (contents|top)|table of contents|skip to (main content|navigation))\\b/gi, '') // Remove common navigation phrases\n .trim(); // Remove leading and trailing whitespace\n}\n\n// Updated function to fetch and parse URL content\nasync function fetchUrlContent(url) {\n try {\n console.log('Fetching URL:', url);\n const response = await fetch(url);\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n const html = await response.text();\n console.log('Fetched HTML length:', html.length);\n const root = parse(html);\n\n if (!root || typeof root.querySelectorAll !== 'function') {\n throw new Error('Failed to parse HTML content');\n }\n\n // Remove script, style, nav, header, footer, and other non-content elements\n const elementsToRemove = root.querySelectorAll('script, style, nav, header, footer, aside, .navigation, .menu, .sidebar, [role=\"navigation\"], .ad, .advertisement, .social-share, .related-articles, .comments, .author-bio');\n elementsToRemove.forEach(el => el.remove());\n\n // Extract content from the main content area if possible, otherwise use body\n const mainContent = root.querySelector('main, #main, #content, .content, article, [role=\"main\"]') || root.querySelector('body');\n let content = mainContent ? mainContent.textContent : '';\n\n if (!content) {\n console.log('No content found in parsed HTML, falling back to full body content');\n content = root.querySelector('body').textContent;\n }\n\n console.log('Extracted content length:', content.length);\n\n // Clean the extracted text\n content = cleanText(content);\n console.log('Cleaned content length:', content.length);\n\n if (content.trim().length === 0) {\n throw new Error('Cleaned content is empty');\n }\n\n return content;\n } catch (error) {\n console.error('Error fetching URL:', error);\n throw new Error(`Failed to fetch URL content: ${error.message}`);\n }\n}\n\napp.get(\"/\", (c) => {\n return c.html(`\n \n \n \n \n \n Radical Text Analyser\n \n \n \n \n \n \n \n

\n Welcome to the Radical Text Analyser, mate!\n

Radical Text Analyser

Enter your text or a URL below to get a totally brilliant analysis, mate!

\n \n

Analysing... Hang tight, mate!

\n $\"Under$ \n This site is always under construction!\n

TF-IDF (Term Frequency-Inverse Document Frequency) is a brilliant way to figure out how important a word or phrase is in a document. Higher scores mean more important words or phrases!

We show the top 7 TF-IDF scores because it's a good balance between providing enough information and not overwhelming the user. It's also considered a \"lucky\" number in many cultures!

\n View Source\n

\n \n \n \n `);\n});\n\napp.post('/analyse', async (c) => {\n const { text } = await c.req.json();\n \n if (text.length > 100000) {\n return c.json({ error: \"Text too long. Please limit to 100,000 characters.\" }, 400);\n }\n\n const words = text.trim().split(/\\s+/).length;\n const characters = text.length;\n\n // Calculate TF-IDF\n const tfidfScores = calculateTFIDF(text, 7);\n\n // Calculate word frequency\n const frequency = calculateWordFrequency(text, 30);\n\n // Generate fun fact\n const funFact = generateFunFact(text);\n\n // Perform sentiment analysis\n const sentimentAnalysis = analyseSentiment(text);\n\n // Generate summary\n const summary = generateSummary(text);\n\n return c.json({ words, characters, tfidf: tfidfScores, frequency, funFact, sentiment: sentimentAnalysis, summary });\n});\n\napp.post('/fetch-url', async (c) => {\n const { url } = await c.req.json();\n \n try {\n console.log('Fetching URL:', url);\n const content = await fetchUrlContent(url);\n console.log('Fetched content length:', content.length);\n if (content.length > 100000) {\n return c.json({ error: \"Content from URL is too long. Please try a different URL with less content.\" }, 400);\n }\n if (content.trim().length === 0) {\n return c.json({ error: \"The fetched content is empty. Please try a different URL.\" }, 400);\n }\n return c.json({ content });\n } catch (error) {\n console.error('Error fetching URL:', error);\n return c.json({ error: error.message }, 400);\n }\n});\n\nexport default async function(request: Request): Promise {\n return app.fetch(request);\n}","import { Hono } from \"https://esm.sh/hono@3.10.2\";\nimport { encode } from \"https://esm.sh/js-base64@3.7.5\";\nimport { parse } from \"https://esm.sh/node-html-parser@6.1.5\";\n\nconst app = new Hono();\n\n// Expanded stopwords list with additional web-specific terms\nconst stopwords = new Set([\n 'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'because',\n 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'could', 'did', 'do', 'does', 'doing', 'down',\n 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'has', 'have', 'having', 'he', 'her', 'here', 'hers',\n 'herself', 'him', 'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'its', 'itself', 'just', 'me',\n 'more', 'most', 'my', 'myself', 'no', 'nor', 'not', 'now', 'of', 'off', 'on', 'once', 'only', 'or', 'other',\n 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some', 'such', 'than',\n 'that', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'these', 'they', 'this', 'those', 'through',\n 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which', 'while', 'who',\n 'whom', 'why', 'with', 'would', 'you', 'your', 'yours', 'yourself', 'yourselves',\n 'back', 'contents', 'menu', 'home', 'next', 'previous', 'top', 'bottom', 'navigation', 'nav', 'sidebar', 'footer',\n 'header', 'main', 'article', 'section', 'page', 'skip', 'jump', 'return',\n 'image', 'photo', 'photograph', 'picture', 'fullscreen', 'share', 'reuse', 'features', 'topics', 'observer',\n 'getty', 'rex', 'shutterstock', 'AP', 'AFP', 'reuters', 'EPA', 'PA', 'alamy', 'getty images', 'associated press',\n 'Agence France-Presse', 'Press Association', 'copyright', 'rights reserved', 'all rights reserved',\n 'click', 'tap', 'swipe', 'scroll', 'zoom', 'enlarge', 'expand', 'collapse', 'hide', 'show',\n 'login', 'sign in', 'register', 'subscribe', 'newsletter', 'email', 'follow', 'like', 'comment', 'share',\n 'twitter', 'facebook', 'Instagram', 'LinkedIn', 'Pinterest', 'YouTube', 'Google',\n 'advertisement', 'sponsored', 'promoted', 'partner content', 'paid content',\n 'cookie', 'privacy', 'terms', 'conditions', 'policy', 'agreement',\n 'mobile', 'desktop', 'tablet', 'app', 'website', 'browser',\n 'search', 'find', 'results', 'query', 'keyword',\n 'loading', 'please wait', 'error', 'not found', '404', 'server error', '500',\n 'menu', 'navigation', 'sitemap', 'archive', 'category', 'tag', 'related', 'popular', 'trending',\n 'breaking news', 'latest', 'update', 'live', 'developing story',\n 'read more', 'continue reading', 'full article', 'paywall', 'subscribe now',\n 'comments', 'discussion', 'forum', 'community', 'join the conversation',\n 'share your thoughts', 'leave a comment', 'what do you think',\n 'fact check', 'analysis', 'opinion', 'editorial', 'column', 'blog post',\n 'video', 'audio', 'podcast', 'gallery', 'slideshow', 'interactive',\n 'download', 'upload', 'attachment', 'file', 'document',\n 'print', 'email', 'save', 'bookmark', 'favorite',\n 'font size', 'text size', 'dark mode', 'light mode', 'accessibility',\n 'support', 'help', 'faq', 'contact us', 'about us', 'team', 'careers',\n 'advertise', 'partnerships', 'sponsorships', 'media kit',\n 'rss feed', 'xml', 'api', 'widget', 'embed',\n 'copyright', 'trademark', 'legal', 'disclaimer', 'terms of use', 'privacy policy'\n]);\n\n// Expanded Synonym dictionary (unchanged)\nconst synonyms = {\n \"important\": [\"significant\", \"crucial\", \"vital\", \"essential\", \"key\"],\n // ... (rest of the synonyms dictionary remains unchanged)\n};\n\n// Simple sentiment analysis function (unchanged)\nfunction analyseSentiment(text) {\n const positiveWords = new Set([\n 'good', 'great', 'excellent', 'wonderful', 'amazing', 'fantastic', 'awesome',\n 'happy', 'joy', 'love', 'like', 'beautiful', 'nice', 'pleasant', 'delightful'\n ]);\n const negativeWords = new Set([\n 'bad', 'terrible', 'awful', 'horrible', 'dreadful', 'poor', 'disappointing',\n 'sad', 'unhappy', 'hate', 'dislike', 'ugly', 'unpleasant', 'nasty', 'cruel'\n ]);\n\n const words = text.toLowerCase().match(/\\b\\w+\\b/g) || [];\n let score = 0;\n words.forEach(word => {\n if (positiveWords.has(word)) score++;\n if (negativeWords.has(word)) score--;\n });\n\n let sentiment;\n if (score > 0) {\n sentiment = \"positive\";\n } else if (score < 0) {\n sentiment = \"negative\";\n } else {\n sentiment = \"neutral\";\n }\n\n return { score, sentiment };\n}\n\n// Improved TF-IDF implementation with additional filtering\nfunction calculateTFIDF(text, outputSize = 7) {\n const words = text.toLowerCase().match(/\\b[a-z]{2,}\\b/g) || [];\n const wordCount = words.length;\n const termFrequency = {};\n const documentFrequency = {};\n\n // Calculate term frequency and document frequency\n for (let i = 0; i < words.length; i++) {\n for (let j = 1; j <= 3 && i + j <= words.length; j++) {\n const phrase = words.slice(i, i + j).join(' ');\n if (!phrase.split(' ').every(word => stopwords.has(word))) {\n termFrequency[phrase] = (termFrequency[phrase] || 0) + 1;\n documentFrequency[phrase] = 1;\n }\n }\n }\n\n // Calculate TF-IDF with improved scoring\n const tfidf = {};\n Object.keys(termFrequency).forEach(term => {\n const tf = termFrequency[term] / wordCount;\n const idf = Math.log(wordCount / (documentFrequency[term] || 1)) + 1;\n const termWordCount = term.split(' ').length;\n tfidf[term] = tf * idf * (termWordCount * 0.5) * (1 / Math.sqrt(termFrequency[term])) * (term.length / 10);\n });\n\n // Sort by score (descending) and get top unique phrases\n const uniquePhrases = new Set();\n return Object.entries(tfidf)\n .sort((a, b) => b[1] - a[1])\n .filter(([phrase, _]) => {\n if (uniquePhrases.size >= outputSize) return false;\n const words = phrase.split(' ');\n if (words.length > 3) return false;\n if (words.some(word => word.length < 3)) return false;\n if (words.every(word => stopwords.has(word))) return false;\n if (Array.from(uniquePhrases).some(existingPhrase => existingPhrase.includes(phrase))) return false;\n if (phrase.includes('http') || phrase.includes('www') || phrase.includes('com')) return false;\n if (/\\d/.test(phrase)) return false; // Exclude phrases containing numbers\n if (phrase.length < 4) return false; // Exclude very short phrases\n uniquePhrases.add(phrase);\n return true;\n })\n .slice(0, outputSize);\n}\n\n// Updated function to calculate word frequency\nfunction calculateWordFrequency(text, topN = 30) {\n const words = text.toLowerCase().match(/\\b[a-z]{3,}\\b/g) || [];\n const frequency = {};\n words.forEach(word => {\n if (!stopwords.has(word) && !word.includes('http') && !word.includes('www') && !word.includes('com')) {\n frequency[word] = (frequency[word] || 0) + 1;\n }\n });\n return Object.entries(frequency)\n .sort((a, b) => b[1] - a[1])\n .slice(0, topN);\n}\n\n// Updated function to generate a fun fact\nfunction generateFunFact(text) {\n const wordCount = text.trim().split(/\\s+/).length;\n const charCount = text.length;\n const sentences = text.split(/[.!?]+/).length;\n const avgWordLength = wordCount > 0 ? charCount / wordCount : 0;\n\n const words = text.match(/\\b\\w+\\b/g) || [];\n const longestWord = words.length > 0 ? words.reduce((a, b) => a.length > b.length ? a : b) : 'N/A';\n\n const facts = [\n `Your text has an average word length of ${avgWordLength.toFixed(2)} characters. Shakespeare's average was about 3.99!`,\n `You've written ${sentences} sentence${sentences !== 1 ? 's' : ''}. A typical novel has about 50-60 sentences per page.`,\n `Your text is ${wordCount} words long. \"War and Peace\" by Leo Tolstoy is about 587,287 words!`,\n `Your longest word is \"${longestWord}\". The longest word in English is 189,819 letters long!`,\n `You've used ${charCount} characters. That's about ${(charCount / 280).toFixed(2)} tweets' worth of content!`\n ];\n\n return facts[Math.floor(Math.random() * facts.length)];\n}\n\n// Function to generate a simple summary (unchanged)\nfunction generateSummary(text, sentenceCount = 3) {\n const sentences = text.match(/[^\\.!\\?]+[\\.!\\?]+/g) || [];\n return sentences.slice(0, sentenceCount).join(' ');\n}\n\n// Updated function to clean the extracted text\nfunction cleanText(text) {\n return text\n .replace(/<[^>]*>/g, '') // Remove HTML tags\n .replace(/\\s+/g, ' ') // Replace multiple spaces with a single space\n .replace(/[^\\w\\s.,!?-]/g, ' ') // Replace special characters with spaces, except for some punctuation\n .replace(/\\b(https?:\\/\\/)?[^\\s]+\\.(com|org|net|edu|gov)\\b/g, '') // Remove URLs\n .replace(/\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b/g, '') // Remove email addresses\n .replace(/\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\b/g, '') // Remove IP addresses\n .replace(/\\b(back to (contents|top)|table of contents|skip to (main content|navigation))\\b/gi, '') // Remove common navigation phrases\n .trim(); // Remove leading and trailing whitespace\n}\n\n// Updated function to fetch and parse the text\nasync function fetchUrlContent(url) {\n try {\n console.log('Fetching URL:', url);\n const response = await fetch(url);\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n const html = await response.text();\n console.log('Fetched HTML length:', html.length);\n const root = parse(html);\n\n if (!root || typeof root.querySelectorAll !== 'function') {\n throw new Error('Failed to parse HTML content');\n }\n\n// Remove script, style, nav, header, footer, and other non-content elements\nconst elementsToRemove = root.querySelectorAll('script, style, nav, header, footer, aside, .navigation, .menu, .sidebar, [role=\"navigation\"], .ad, .advertisement, .social-share, .related-articles, .comments, .author-bio');\nelementsToRemove.forEach(el => el.remove());\n\n// Extract content from the main content area if possible, otherwise use body\nconst mainContent = root.querySelector('main, #main, #content, .content, article, [role=\"main\"]') || root.querySelector('body');\nlet content = mainContent ? mainContent.textContent : '';\n\nif (!content) {\n console.log('No content found in parsed HTML, falling back to full body content');\n content = root.querySelector('body').textContent;\n}\n\nconsole.log('Extracted content length:', content.length);\n\n// Clean the extracted text\ncontent = cleanText(content);\nconsole.log('Cleaned content length:', content.length);\n\nif (content.trim().length === 0) {\n throw new Error('Cleaned content is empty');\n}\n\nreturn content;\n} catch (error) {\n console.error('Error fetching URL:', error);\n throw new Error(`Failed to fetch URL content: ${error.message}`);\n}\n}\n\napp.get(\"/\", (c) => {\n return c.html(`\n \n \n \n \n \n Radical Text Analyser\n \n \n \n \n \n \n \n

\n Welcome to the Radical Text Analyser, mate!\n

Radical Text Analyser

Enter your text or a URL below to get a totally brilliant analysis, mate!

\n \n

Analysing... Hang tight, mate!

\n $\"Under$ \n This site is always under construction!\n

TF-IDF (Term Frequency-Inverse Document Frequency) is a brilliant way to figure out how important a word or phrase is in a document. Higher scores mean more important words or phrases!

We show the top 7 TF-IDF scores because it's a good balance between providing enough information and not overwhelming the user. It's also considered a \"lucky\" number in many cultures!

\n View Source\n

\n \n \n \n `);\n});\n\napp.post('/analyse', async (c) => {\n const { text } = await c.req.json();\n \n if (text.length > 100000) {\n return c.json({ error: \"Text too long. Please limit to 100,000 characters.\" }, 400);\n }\n\n const words = text.trim().split(/\\s+/).length;\n const characters = text.length;\n\n // Calculate TF-IDF\n const tfidfScores = calculateTFIDF(text, 7);\n\n // Calculate word frequency\n const frequency = calculateWordFrequency(text, 30);\n\n // Generate fun fact\n const funFact = generateFunFact(text);\n\n // Perform sentiment analysis\n const sentimentAnalysis = analyseSentiment(text);\n\n // Generate summary\n const summary = generateSummary(text);\n\n return c.json({ words, characters, tfidf: tfidfScores, frequency, funFact, sentiment: sentimentAnalysis, summary });\n});\n\napp.post('/fetch-url', async (c) => {\n const { url } = await c.req.json();\n \n try {\n console.log('Fetching URL:', url);\n const content = await fetchUrlContent(url);\n console.log('Fetched content length:', content.length);\n if (content.length > 100000) {\n return c.json({ error: \"Content from URL is too long. Please try a different URL with less content.\" }, 400);\n }\n if (content.trim().length === 0) {\n return c.json({ error: \"The fetched content is empty. Please try a different URL.\" }, 400);\n }\n return c.json({ content });\n } catch (error) {\n console.error('Error fetching URL:', error);\n return c.json({ error: error.message }, 400);\n }\n});\n\nexport default async function(request: Request): Promise {\n return app.fetch(request);\n}","createdAt",["D",1731719894609],"branchVersion","branchId","6bc89b82-2794-11f0-9d52-569c3dd06744","id","6bc837e6-2794-11f0-9d52-569c3dd06744","sanguineCyanMastodon",["D",1746217886539],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","aca71035-c107-41d7-85c9-3549aca42d8f","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","7545e6dc-57fc-46b5-8123-3cf67ef161b9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":54,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"isOwned",109,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","mikehiggins","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yb1pVTEptempxRWlwOGZVUW5kV2NRdWdxVzAifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":160,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":31,"_56":57,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":161,"_79":162,"_81":163,"_83":25},"tree",[151,152],"files",[151,152],"counts",{"_141":61,"_142":137,"_143":61,"_144":145,"_146":147},"contents",[125,126],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":134,"_32":135,"_30":136,"_128":-5,"_41":42,"_40":137,"_129":57,"_38":138,"_13":61,"_33":20,"_35":139,"_24":25,"_132":-5,"_27":140},{"_43":127,"_32":28,"_30":31,"_128":-5,"_41":42,"_40":14,"_129":25,"_38":130,"_13":131,"_33":34,"_35":37,"_24":25,"_132":-5,"_27":133},"6db0d0ae-2794-11f0-9d52-569c3dd06744","parentId","latest",["D",1731719894609],80,"label",[28],"6bcabd86-2794-11f0-9d52-569c3dd06744","README.md","6bcabe26-2794-11f0-9d52-569c3dd06744",1,["D",1731086850651],"Project Name: Radical Text Analyser\nPrimary Purpose: A web-based text analysis tool that counts words, calculates TF-IDF, analyzes sentiment, and generates fun facts about the input text.\nMain Technologies: Hono framework, HTML parsing, JS Base64 encoding, and Node.js.\n\nWord and character counts\nTF-IDF score calculation for the importance of phrases\nWord frequency analysis\nBasic sentiment analysis (positive, negative, or neutral)\nFun fact generation about the text (e.g., average word length)\nSummary generation\nThe tool is implemented in JavaScript using Hono as a lightweight framework and integrates web APIs for fetching external content and analyzing it in real-time.\n\nCode Structure\nImports\n\nHono: Used for routing and handling API requests.\nencode (from JS Base64): Utility for encoding data.\nparse (from Node HTML Parser): Parses HTML content, used in fetchUrlContent to clean and extract text.\nCore Modules\n\napp: Initializes the main app instance with Hono.\nstopwords: Contains a list of common stop words and web-specific terms to filter out during text analysis.\nsynonyms: Dictionary mapping common words to their synonyms for highlighting in TF-IDF analysis.\nFunctions\nThe primary functions perform specific text analysis tasks, as detailed in the following sections.\n\nFunction Documentation\nEach function's purpose, parameters, and output are described below, along with notes on how they are used within the application.\n\nanalyseSentiment(text)\nPurpose: Analyzes the sentiment of a given text.\nParameters:\n\ntext (string): The text to be analyzed. Returns: An object with score (numerical sentiment score) and sentiment (positive, neutral, or negative).\nUsage: Called within the /analyse endpoint to evaluate the overall sentiment of the user’s input.\ncalculateTFIDF(text, outputSize=7)\nPurpose: Calculates TF-IDF scores to measure the importance of words/phrases in the text.\nParameters:\n\ntext (string): The text for analysis.\noutputSize (number, optional): Limits the number of phrases to return. Returns: Array of top phrases with scores, filtered for uniqueness and length constraints.\nUsage: Called in /analyse to highlight significant terms.\ncalculateWordFrequency(text, topN=30)\nPurpose: Counts word frequency, excluding stop words and web-specific terms.\nParameters:\n\ntext (string): Input text.\ntopN (number, optional): Number of top words to return. Returns: Array of most frequent words with counts.\nUsage: Used within /analyse to populate the word frequency analysis section.\ngenerateFunFact(text)\nPurpose: Generates a fun fact based on text statistics (e.g., word length).\nParameters:\n\ntext (string): Text for analysis. Returns: String with a randomly chosen fun fact.\nUsage: Triggered in /analyse to add a unique, informational element to results.\ngenerateSummary(text, sentenceCount=3)\nPurpose: Extracts a summary from the first few sentences of the input.\nParameters:\n\ntext (string): Text for summarization.\nsentenceCount (number, optional): Number of sentences in the summary. Returns: Summary string.\nUsage: Included in /analyse for quick overviews.\ncleanText(text)\nPurpose: Cleans HTML and unwanted characters from the fetched content.\nParameters:\n\ntext (string): Raw HTML/text. Returns: Cleaned text string.\nUsage: Essential for preprocessing content in fetchUrlContent.\nfetchUrlContent(url)\nPurpose: Fetches HTML content from a URL, removes unwanted elements, and extracts main text.\nParameters:\n\nurl (string): URL to fetch and clean content from. Returns: Cleaned text string or an error message.\nUsage: This function powers the URL input functionality.\nCore Functionalities\nText Analysis Form (HTML)\nThe form on the main page allows users to submit either raw text or a URL. The client-side JavaScript processes the form submission and, if needed, triggers a URL fetch to obtain content.\n\nBackend Processing (Routes)\n\nGET /: Serves the main HTML page.\nPOST /analyse: Analyzes submitted text and returns word count, sentiment, summary, TF-IDF, and word frequency results.\nPOST /fetch-url: Retrieves and cleans content from a URL, then prepares it for analysis.\nDark Mode and Interactivity\n\nDark Mode Toggle: Client-side JavaScript manages style changes.\nShare Feature: Allows users to copy a link with encoded text or URL.\nWord Cloud: jQCloud generates a word cloud based on frequency data.\nMaintenance Notes\nTesting\n\nIntegration Tests: Ensure calculateTFIDF, calculateWordFrequency, and fetchUrlContent produce consistent outputs.\nClient-Side Testing: Regularly test URL fetching functionality to ensure robustness, particularly with dynamic sites.\nError Handling\n\nMaintain try/catch blocks in asynchronous functions like fetchUrlContent to capture network issues.\nUpdate error messages to be user-friendly and provide feedback if inputs are invalid or exceed size limits.\nData Sanitization\n\nVerify that cleanText removes sensitive information from fetched URLs to prevent accidental disclosure.\nRate Limits and API Usage\n\nConsider implementing rate limiting or caching mechanisms to manage heavy usage or repeated requests for the same content.\n\nFuture Improvements\n\nEnhanced Synonym Dictionary\n\nExpand synonyms for better word variation support during analysis.\n\nAdvanced Sentiment Analysis\n\nImplement a sentiment analysis library or API for more nuanced scoring and classification.\n\nMultilingual Support\n\nExtend stop words, sentiment analysis, and cleaning functions to support languages beyond English.\n\nThe Radical Text Analyser code primarily uses the following APIs:\n1. Hono Framework API\n * Purpose: Used to set up routing and serve responses for different endpoints (GET /, POST /analyse, and POST /fetch-url).\n * Usage: The Hono framework handles HTTP requests and responses, forming the backbone of the server-side API.\n2. External Content Fetching API (via fetch)\n * Purpose: Retrieves HTML content from external URLs when users input a URL for analysis.\n * Usage: The fetchUrlContent function uses fetch to make a GET request to the user-provided URL, which allows the application to process content from web pages.\n * Error Handling: Includes logic to check response status and handle various network errors.\n3. Google Fonts API\n * Purpose: Loads the \"VT323\" font for styling the front-end.\n * Usage: Linked in the HTML section via .\n4. jQCloud Library (via CDN)\n * Purpose: Used for generating a word cloud from frequency data on the client side.\n * Usage: Loaded in the HTML section with and \n\n The TF-IDF analysis in this code is performed without using an external library. Instead, it’s implemented directly in the `calculateTFIDF` function. Here’s how it works:\n\n1. **Tokenization and Filtering**: The text is split into words and phrases, excluding stopwords and web-specific terms.\n2. **Term Frequency (TF)**: For each unique phrase, the code calculates how frequently it appears in the text relative to the total number of words.\n3. **Inverse Document Frequency (IDF)**: Each term’s IDF is calculated using a formula that reduces the weight of commonly occurring phrases.\n4. **Scoring**: TF and IDF are multiplied, adjusted by phrase length and rarity, to boost significant phrases.\n5. **Output**: Top phrases are sorted by score, filtered for uniqueness, and returned as the most relevant terms. \n\nThis custom implementation allows for added customization, such as phrase filtering and length adjustments, which wouldn’t be as easily configurable with a standard library.\n\nIDF = ln(Total Word Count / Document Frequency of Term) + 1\n\nTotal Word Count: The total number of words in the input text.\nDocument Frequency of Term: How many times the term appears across documents, which is set to 1 in this single-document context to avoid division by zero.\n+1 Adjustment: Adding 1 ensures that terms appearing once don't have an IDF of zero, which would nullify their score.",[135],"pulls","branches","forks","envVars",{"_148":61,"_150":61},"connections",{"_148":61,"_149":61},"count","unfilledCount","emptyCount",{"_43":134,"_32":135,"_30":136,"_128":-5,"_41":42,"_40":137,"_129":57,"_38":157,"_13":61,"_33":20,"_35":139,"_24":25,"_132":-5,"_154":158,"_27":159},{"_43":127,"_32":28,"_30":31,"_128":-5,"_41":42,"_40":14,"_129":25,"_38":153,"_13":131,"_33":34,"_35":37,"_24":25,"_132":-5,"_154":155,"_27":156},["D",1731719894609],"children",[],[28],["D",1731086850651],[],[135],["D",1746217886539],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":54,"_99":100,"_101":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"ctx",{"_77":-5,"_122":197,"_198":199,"_200":201,"_202":25,"_203":25,"_204":25,"_205":57,"_206":-5,"_207":-7,"_208":-7,"_209":61,"_210":25,"_211":25,"_212":-5,"_213":25,"_214":25,"_215":25,"_216":-5},"ENV",{"_168":169,"_170":171,"_172":85,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192,"_193":194,"_195":196},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","5015b8df46402d8bba4f485ecc37aec202b0e06b","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v81