[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_179":180,"_181":182},"routes/_app.x.$handle.$name",{"_96":95,"_32":45,"_104":86,"_27":105,"_13":14,"_106":-5,"_11":107,"_108":109,"_110":111,"_112":113,"_114":115,"_20":-5,"_116":-5,"_117":-5,"_118":25,"_119":120,"_121":25,"_122":-5,"_123":-5,"_124":125},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":56,"_57":58,"_59":58,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":25},"version",44,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","html-to-markdown.ts","latestPath","fileId","38fe1910-8d0c-11f0-956f-0224a6c84d84","name","fileType","script","content","import TurndownService from \"npm:turndown\";\nimport { MathMLToLaTeX } from \"npm:mathml-to-latex\";\nimport { processUrls } from \"./string-utils.ts\";\nimport { parseHTML } from \"npm:linkedom\";\n// import { debugLog } from './debug';\n\nfunction debugLog(...args) {\n if (import.meta?.env?.NODE_ENV === \"development\") {\n console.log(...args);\n }\n}\n\nconst footnotes: { [key: string]: string } = {};\n\nexport function createMarkdownContent(content: string, url: string) {\n let {\n HTMLElement,\n Element,\n Node,\n HTMLOListElement,\n HTMLTableElement,\n HTMLIFrameElement,\n HTMLPreElement,\n HTMLLiElement,\n HTMLAnchorElement,\n } = parseHTML(content);\n\n debugLog(\"Markdown\", \"Starting markdown conversion for URL:\", url);\n debugLog(\"Markdown\", \"Content length:\", content.length);\n\n const baseUrl = new URL(url);\n // Process all URLs at the beginning\n const processedContent = processUrls(content, baseUrl);\n\n const turndownService = new TurndownService({\n headingStyle: \"atx\",\n hr: \"---\",\n bulletListMarker: \"-\",\n codeBlockStyle: \"fenced\",\n emDelimiter: \"*\",\n preformattedCode: true,\n });\n\n turndownService.addRule(\"table\", {\n filter: \"table\",\n replacement: function (content, node) {\n if (!(node instanceof HTMLTableElement)) return content;\n\n // Check if it's an ArXiv equation table\n if (\n node.classList.contains(\"ltx_equation\") ||\n node.classList.contains(\"ltx_eqn_table\")\n ) {\n return handleNestedEquations(node);\n }\n\n // Check if the table has colspan or rowspan\n const hasComplexStructure = Array.from(node.querySelectorAll(\"td, th\"))\n .some((cell) =>\n cell.hasAttribute(\"colspan\") || cell.hasAttribute(\"rowspan\")\n );\n\n if (hasComplexStructure) {\n // Clean up the table HTML\n const cleanedTable = cleanupTableHTML(node);\n return \"\\n\\n\" + cleanedTable + \"\\n\\n\";\n }\n\n // Process simple tables as before\n const rows = Array.from(node.rows).map((row) => {\n const cells = Array.from(row.cells).map((cell) => {\n // Remove newlines and trim the content\n let cellContent = turndownService.turndown(cell.innerHTML)\n .replace(/\\n/g, \" \")\n .trim();\n // Escape pipe characters\n cellContent = cellContent.replace(/\\|/g, \"\\\\|\");\n return cellContent;\n });\n return `| ${cells.join(\" | \")} |`;\n });\n\n // Create the separator row\n const separatorRow = `| ${\n Array(rows[0].split(\"|\").length - 2).fill(\"---\").join(\" | \")\n } |`;\n\n // Combine all rows\n const tableContent = [rows[0], separatorRow, ...rows.slice(1)].join(\"\\n\");\n\n return `\\n\\n${tableContent}\\n\\n`;\n },\n });\n\n turndownService.remove([\"style\", \"script\", \"button\"]);\n\n // Keep iframes, video, audio, sup, and sub elements\n // @ts-ignore\n turndownService.keep([\n \"iframe\",\n \"video\",\n \"audio\",\n \"sup\",\n \"sub\",\n \"svg\",\n \"math\",\n ]);\n\n turndownService.addRule(\"list\", {\n filter: [\"ul\", \"ol\"],\n replacement: function (content: string, node: Node) {\n // Remove trailing newlines/spaces from content\n content = content.trim();\n\n // Add a newline before the list if it's a top-level list\n const isTopLevel = !(node.parentNode &&\n (node.parentNode.nodeName === \"UL\" ||\n node.parentNode.nodeName === \"OL\"));\n return (isTopLevel ? \"\\n\" : \"\") + content + \"\\n\";\n },\n });\n\n // Lists with tab indentation\n turndownService.addRule(\"listItem\", {\n filter: \"li\",\n replacement: function (\n content: string,\n node: Node,\n options: TurndownService.Options,\n ) {\n if (!(node instanceof HTMLElement) || !(node instanceof HTMLLiElement)) {\n return content;\n }\n\n // Handle task list items\n const isTaskListItem = node.classList.contains(\"task-list-item\");\n const checkbox = node.querySelector('input[type=\"checkbox\"]') as\n | HTMLInputElement\n | null;\n let taskListMarker = \"\";\n\n if (isTaskListItem && checkbox) {\n // Remove the checkbox from content since we'll add markdown checkbox\n content = content.replace(/]*>/, \"\");\n taskListMarker = checkbox.checked ? \"[x] \" : \"[ ] \";\n }\n\n content = content\n // Remove trailing newlines\n .replace(/\\n+$/, \"\")\n // Split into lines\n .split(\"\\n\")\n // Remove empty lines\n .filter((line) => line.length > 0)\n // Add indentation to continued lines\n .join(\"\\n\\t\");\n\n let prefix = options.bulletListMarker + \" \";\n let parent = node.parentNode;\n\n // Calculate the nesting level\n let level = 0;\n let currentParent = node.parentNode;\n while (\n currentParent &&\n (currentParent.nodeName === \"UL\" || currentParent.nodeName === \"OL\")\n ) {\n level++;\n currentParent = currentParent.parentNode;\n }\n\n // Add tab indentation based on nesting level, ensuring it's never negative\n const indentLevel = Math.max(0, level - 1);\n prefix = \"\\t\".repeat(indentLevel) + prefix;\n\n if (parent instanceof HTMLOListElement) {\n let start = parent.getAttribute(\"start\");\n let index = Array.from(parent.children).indexOf(node as HTMLElement) +\n 1;\n prefix = \"\\t\".repeat(level - 1) +\n (start ? Number(start) + index - 1 : index) + \". \";\n }\n\n return prefix + taskListMarker + content.trim() +\n (node.nextSibling && !/\\n$/.test(content) ? \"\\n\" : \"\");\n },\n });\n\n turndownService.addRule(\"figure\", {\n filter: \"figure\",\n replacement: function (content, node) {\n const figure = node as HTMLElement;\n const img = figure.querySelector(\"img\");\n const figcaption = figure.querySelector(\"figcaption\");\n\n if (!img) return content;\n\n const alt = img.getAttribute(\"alt\") || \"\";\n let src = img.getAttribute(\"src\") || \"\";\n const srcset = img.getAttribute(\"srcset\") || \"\";\n\n if (srcset) {\n let images: { src: string; width: number }[] = [];\n\n // deconstruct the srcset urls\n const srcsetUrls = srcset.split(\",%20\");\n for (let url of srcsetUrls) {\n const split = url.split(\"%20\");\n const srcUrl = split[0];\n const width = parseInt(split[1]);\n\n images.push({ src: srcUrl, width });\n }\n\n // we need to choose one of the images to use from the srcset\n // for convenience, simply grab the first one\n // this could be updated to take width into account\n\n src = images[0].src;\n }\n\n let caption = \"\";\n\n if (figcaption) {\n const tagSpan = figcaption.querySelector(\".ltx_tag_figure\");\n const tagText = tagSpan ? tagSpan.textContent?.trim() : \"\";\n\n // Process the caption content, including math elements\n let captionContent = figcaption.innerHTML;\n captionContent = captionContent.replace(\n /(.*?)<\\/math>/g,\n (match, mathContent, offset, string) => {\n const mathElement = new DOMParser().parseFromString(\n match,\n \"text/html\",\n ).body.firstChild as Element;\n const latex = extractLatex(mathElement);\n const prevChar = string[offset - 1] || \"\";\n const nextChar = string[offset + match.length] || \"\";\n\n const isStartOfLine = offset === 0 || /\\s/.test(prevChar);\n const isEndOfLine = offset + match.length === string.length ||\n /\\s/.test(nextChar);\n\n const leftSpace = (!isStartOfLine && !/[\\s$]/.test(prevChar))\n ? \" \"\n : \"\";\n const rightSpace = (!isEndOfLine && !/[\\s$]/.test(nextChar))\n ? \" \"\n : \"\";\n\n return `${leftSpace}$${latex}$${rightSpace}`;\n },\n );\n\n // Convert the processed caption content to markdown\n const captionMarkdown = turndownService.turndown(captionContent);\n\n // Combine tag and processed caption\n caption = `${tagText} ${captionMarkdown}`.trim();\n }\n\n // Handle references in the caption\n caption = caption.replace(\n /\\[([^\\]]+)\\]\$([^)]+)\$/g,\n (match, text, href) => {\n return `[${text}](${href})`;\n },\n );\n\n return `![${alt}](${src})\\n\\n${caption}\\n\\n`;\n },\n });\n\n // Use Obsidian format for YouTube embeds and tweets\n turndownService.addRule(\"embedToMarkdown\", {\n filter: function (node: Node): boolean {\n if (node instanceof HTMLIFrameElement) {\n const src = node.getAttribute(\"src\");\n return !!src && (\n !!src.match(/(?:youtube\\.com|youtu\\.be)/) ||\n !!src.match(/(?:twitter\\.com|x\\.com)/)\n );\n }\n return false;\n },\n replacement: function (content: string, node: Node): string {\n if (node instanceof HTMLIFrameElement) {\n const src = node.getAttribute(\"src\");\n if (src) {\n const youtubeMatch = src.match(\n /(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com|youtu\\.be)\\/(?:embed\\/|watch\\?v=)?([a-zA-Z0-9_-]+)/,\n );\n if (youtubeMatch && youtubeMatch[1]) {\n return `![](https://www.youtube.com/watch?v=${youtubeMatch[1]})`;\n }\n const tweetMatch = src.match(\n /(?:twitter\\.com|x\\.com)\\/.*?(?:status|statuses)\\/(\\d+)/,\n );\n if (tweetMatch && tweetMatch[1]) {\n return `![](https://x.com/i/status/${tweetMatch[1]})`;\n }\n }\n }\n return content;\n },\n });\n\n turndownService.addRule(\"highlight\", {\n filter: \"mark\",\n replacement: function (content) {\n return \"==\" + content + \"==\";\n },\n });\n\n turndownService.addRule(\"strikethrough\", {\n filter: (node: Node) =>\n node.nodeName === \"DEL\" ||\n node.nodeName === \"S\" ||\n node.nodeName === \"STRIKE\",\n replacement: function (content) {\n return \"~~\" + content + \"~~\";\n },\n });\n\n // Add a new custom rule for complex link structures\n turndownService.addRule(\"complexLinkStructure\", {\n filter: function (node, options) {\n return (\n node.nodeName === \"A\" &&\n node.childNodes.length > 1 &&\n Array.from(node.childNodes).some((child) =>\n [\"H1\", \"H2\", \"H3\", \"H4\", \"H5\", \"H6\"].includes(child.nodeName)\n )\n );\n },\n replacement: function (content, node, options) {\n if (\n !(node instanceof HTMLElement) || !(node instanceof HTMLAnchorElement)\n ) return content;\n const href = node.getAttribute(\"href\");\n const title = node.getAttribute(\"title\");\n\n // Extract the heading\n const headingNode = node.querySelector(\"h1, h2, h3, h4, h5, h6\");\n const headingContent = headingNode\n ? turndownService.turndown(headingNode.innerHTML)\n : \"\";\n\n // Remove the heading from the content\n if (headingNode) {\n headingNode.remove();\n }\n\n // Convert the remaining content\n const remainingContent = turndownService.turndown(node.innerHTML);\n\n // Construct the new markdown\n let markdown = `${headingContent}\\n\\n${remainingContent}\\n\\n`;\n if (href) {\n markdown += `[View original](${href})`;\n if (title) {\n markdown += ` \"${title}\"`;\n }\n }\n\n return markdown;\n },\n });\n\n turndownService.addRule(\"arXivEnumerate\", {\n filter: (node) => {\n return node.nodeName === \"OL\" && node.classList.contains(\"ltx_enumerate\");\n },\n replacement: function (content, node) {\n if (\n !(node instanceof HTMLElement) || !(node instanceof HTMLOListElement)\n ) return content;\n\n const items = Array.from(node.children).map((item, index) => {\n if (item instanceof HTMLElement) {\n const itemContent = item.innerHTML.replace(\n /^\\d+\\.<\\/span>\\s*/,\n \"\",\n );\n return `${index + 1}. ${turndownService.turndown(itemContent)}`;\n }\n return \"\";\n });\n\n return \"\\n\\n\" + items.join(\"\\n\\n\") + \"\\n\\n\";\n },\n });\n\n turndownService.addRule(\"removeHiddenElements\", {\n filter: function (node) {\n return (\n node.style.display === \"none\"\n );\n },\n replacement: function () {\n return \"\";\n },\n });\n\n turndownService.addRule(\"citations\", {\n filter: (node: Node): boolean => {\n if (node instanceof Element) {\n return (\n node.nodeName === \"SUP\" && node.id.startsWith(\"fnref:\")\n );\n }\n return false;\n },\n replacement: (content, node) => {\n if (node instanceof HTMLElement) {\n if (node.nodeName === \"SUP\" && node.id.startsWith(\"fnref:\")) {\n const id = node.id.replace(\"fnref:\", \"\");\n // Extract only the primary number before any hyphen\n const primaryNumber = id.split(\"-\")[0];\n return `[^${primaryNumber}]`;\n }\n }\n return content;\n },\n });\n\n // Footnotes list\n turndownService.addRule(\"footnotesList\", {\n filter: (node: Node): boolean => {\n if (node instanceof HTMLOListElement) {\n return (\n node.parentElement?.id === \"footnotes\"\n );\n }\n return false;\n },\n replacement: (content, node) => {\n if ((node instanceof HTMLElement) || (node instanceof HTMLOListElement)) {\n const references = Array.from(node.children).map((li) => {\n let id;\n if (li.id.startsWith(\"fn:\")) {\n id = li.id.replace(\"fn:\", \"\");\n } else {\n const match = li.id.split(\"/\").pop()?.match(/cite_note-(.+)/);\n id = match ? match[1] : li.id;\n }\n\n // Remove the leading sup element if its content matches the footnote id\n const supElement = li.querySelector(\"sup\");\n if (supElement && supElement.textContent?.trim() === id) {\n supElement.remove();\n }\n\n const referenceContent = turndownService.turndown(li.innerHTML);\n // Remove the backlink from the footnote content\n const cleanedContent = referenceContent.replace(/\\s*↩︎$/, \"\").trim();\n return `[^${id.toLowerCase()}]: ${cleanedContent}`;\n });\n return \"\\n\\n\" + references.join(\"\\n\\n\") + \"\\n\\n\";\n }\n return content;\n },\n });\n\n // General removal rules for varous website elements\n turndownService.addRule(\"removals\", {\n filter: function (node) {\n if (!(node instanceof HTMLElement)) return false;\n // Remove the Defuddle backlink from the footnote content\n if (node.getAttribute(\"href\")?.includes(\"#fnref\")) return true;\n if (node.classList.contains(\"footnote-backref\")) return true;\n return false;\n },\n replacement: function (content, node) {\n return \"\";\n },\n });\n\n turndownService.addRule(\"handleTextNodesInTables\", {\n filter: function (node: Node): boolean {\n return node.nodeType === Node.TEXT_NODE &&\n node.parentNode !== null &&\n node.parentNode.nodeName === \"TD\";\n },\n replacement: function (content: string): string {\n return content;\n },\n });\n\n turndownService.addRule(\"preformattedCode\", {\n filter: (node) => {\n return node.nodeName === \"PRE\";\n },\n replacement: (content, node) => {\n // if (!(node instanceof HTMLElement) || !(node instanceof HTMLPreElement)) {\n // console.log(node);\n // return content;\n // }\n\n const codeElement = node.querySelector(\"code\");\n if (!codeElement) {\n return content;\n }\n\n const language = codeElement.getAttribute(\"data-lang\") || \"\";\n const code = codeElement.textContent || \"\";\n\n // Clean up the content and escape backticks\n const cleanCode = code\n .trim()\n .replace(/`/g, \"\\\\`\");\n\n return `\\n\\`\\`\\`${language}\\n${cleanCode}\\n\\`\\`\\`\\n`;\n },\n });\n\n turndownService.addRule(\"MathJax\", {\n filter: (node) => {\n const isMjxContainer = node.nodeName.toLowerCase() === \"mjx-container\";\n return isMjxContainer;\n },\n replacement: (content, node) => {\n if (!(node instanceof HTMLElement)) {\n return content;\n }\n\n const assistiveMml = node.querySelector(\"mjx-assistive-mml\");\n if (!assistiveMml) {\n return content;\n }\n\n const mathElement = assistiveMml.querySelector(\"math\");\n if (!mathElement) {\n return content;\n }\n\n let latex;\n try {\n latex = MathMLToLaTeX.convert(mathElement.outerHTML);\n } catch (error) {\n console.error(\"Error converting MathML to LaTeX:\", error);\n return content;\n }\n\n // Check if it's an inline or block math element\n const isBlock = mathElement.getAttribute(\"display\") === \"block\";\n\n if (isBlock) {\n return `\\n$$\\n${latex}\\n$$\\n`;\n } else {\n return `$${latex}$`;\n }\n },\n });\n\n turndownService.addRule(\"math\", {\n filter: (node) => {\n return node.nodeName.toLowerCase() === \"math\" ||\n (node instanceof Element && node.classList &&\n (node.classList.contains(\"mwe-math-element\") ||\n node.classList.contains(\"mwe-math-fallback-image-inline\") ||\n node.classList.contains(\"mwe-math-fallback-image-display\")));\n },\n replacement: (content, node) => {\n if (!(node instanceof Element)) return content;\n\n let latex = extractLatex(node);\n\n // Remove leading and trailing whitespace\n latex = latex.trim();\n\n // Check if the math element is within a table\n const isInTable = node.closest(\"table\") !== null;\n\n // Check if it's an inline or block math element\n if (\n !isInTable && (\n node.getAttribute(\"display\") === \"block\" ||\n node.classList.contains(\"mwe-math-fallback-image-display\") ||\n (node.parentElement &&\n node.parentElement.classList.contains(\"mwe-math-element\") &&\n node.parentElement.previousElementSibling &&\n node.parentElement.previousElementSibling.nodeName.toLowerCase() ===\n \"p\")\n )\n ) {\n return `\\n$$\\n${latex}\\n$$\\n`;\n } else {\n // For inline math, ensure there's a space before and after only if needed\n const prevNode = node.previousSibling;\n const nextNode = node.nextSibling;\n const prevChar = prevNode?.textContent?.slice(-1) || \"\";\n const nextChar = nextNode?.textContent?.[0] || \"\";\n\n const isStartOfLine = !prevNode ||\n (prevNode.nodeType === Node.TEXT_NODE &&\n prevNode.textContent?.trim() === \"\");\n const isEndOfLine = !nextNode ||\n (nextNode.nodeType === Node.TEXT_NODE &&\n nextNode.textContent?.trim() === \"\");\n\n const leftSpace =\n (!isStartOfLine && prevChar && !/[\\s$]/.test(prevChar)) ? \" \" : \"\";\n const rightSpace = (!isEndOfLine && nextChar && !/[\\s$]/.test(nextChar))\n ? \" \"\n : \"\";\n\n return `${leftSpace}$${latex}$${rightSpace}`;\n }\n },\n });\n\n turndownService.addRule(\"katex\", {\n filter: (node) => {\n return node instanceof HTMLElement &&\n (node.classList.contains(\"math\") || node.classList.contains(\"katex\"));\n },\n replacement: (content, node) => {\n if (!(node instanceof HTMLElement)) return content;\n\n // Try to find the original LaTeX content\n // 1. Check data-latex attribute\n let latex = node.getAttribute(\"data-latex\");\n\n // 2. If no data-latex, try to get from .katex-mathml\n if (!latex) {\n const mathml = node.querySelector(\n '.katex-mathml annotation[encoding=\"application/x-tex\"]',\n );\n latex = mathml?.textContent || \"\";\n }\n\n // 3. If still no content, use text content as fallback\n if (!latex) {\n latex = node.textContent?.trim() || \"\";\n }\n\n // Determine if it's an inline formula\n const mathElement = node.querySelector(\".katex-mathml math\");\n const isInline = node.classList.contains(\"math-inline\") ||\n (mathElement && mathElement.getAttribute(\"display\") !== \"block\");\n\n if (isInline) {\n return `$${latex}$`;\n } else {\n return `\\n$$\\n${latex}\\n$$\\n`;\n }\n },\n });\n\n turndownService.addRule(\"callout\", {\n filter: (node) => {\n return (\n node.nodeName.toLowerCase() === \"div\" &&\n node.classList.contains(\"markdown-alert\")\n );\n },\n replacement: (content, node) => {\n const element = node as HTMLElement;\n\n // Get alert type from the class (e.g., markdown-alert-note -> NOTE)\n const alertClasses = Array.from(element.classList);\n const typeClass = alertClasses.find((c) =>\n c.startsWith(\"markdown-alert-\") && c !== \"markdown-alert\"\n );\n const type = typeClass\n ? typeClass.replace(\"markdown-alert-\", \"\").toUpperCase()\n : \"NOTE\";\n\n // Find the title element and content\n const titleElement = element.querySelector(\".markdown-alert-title\");\n const contentElement = element.querySelector(\n \"p:not(.markdown-alert-title)\",\n );\n\n // Extract content, removing the title from it if present\n let alertContent = content;\n if (titleElement && titleElement.textContent) {\n alertContent = contentElement?.textContent ||\n content.replace(titleElement.textContent, \"\");\n }\n\n // Format as Obsidian callout\n return `\\n> [!${type}]\\n> ${\n alertContent.trim().replace(/\\n/g, \"\\n> \")\n }\\n`;\n },\n });\n\n function handleNestedEquations(table: Element): string {\n const mathElements = table.querySelectorAll(\"math[alttext]\");\n if (mathElements.length === 0) return \"\";\n\n return Array.from(mathElements).map((mathElement) => {\n const alttext = mathElement.getAttribute(\"alttext\");\n if (alttext) {\n // Check if it's an inline or block equation\n const isInline = mathElement.closest(\".ltx_eqn_inline\") !== null;\n return isInline ? `$${alttext.trim()}$` : `\\n$$\\n${alttext.trim()}\\n$$`;\n }\n return \"\";\n }).join(\"\\n\\n\");\n }\n\n function cleanupTableHTML(table: HTMLTableElement): string {\n const allowedAttributes = [\n \"src\",\n \"href\",\n \"style\",\n \"align\",\n \"width\",\n \"height\",\n \"rowspan\",\n \"colspan\",\n \"bgcolor\",\n \"scope\",\n \"valign\",\n \"headers\",\n ];\n\n const cleanElement = (element: Element) => {\n Array.from(element.attributes).forEach((attr) => {\n if (!allowedAttributes.includes(attr.name)) {\n element.removeAttribute(attr.name);\n }\n });\n\n element.childNodes.forEach((child) => {\n if (child instanceof Element) {\n cleanElement(child);\n }\n });\n };\n\n // Create a clone of the table to avoid modifying the original DOM\n const tableClone = table.cloneNode(true) as HTMLTableElement;\n cleanElement(tableClone);\n\n return tableClone.outerHTML;\n }\n\n function extractLatex(element: Element): string {\n // Check if the element is a

element and has an alttext attribute\n    if (element.nodeName.toLowerCase() === \"math\") {\n      let latex = element.getAttribute(\"data-latex\");\n      let alttext = element.getAttribute(\"alttext\");\n      if (latex) {\n        return latex.trim();\n      } else if (alttext) {\n        return alttext.trim();\n      }\n      console.log(\"No latex or alttext found for math element:\", element);\n    }\n\n    // If not, look for a nested element with alttext\n    const mathElement = element.querySelector(\"math[alttext]\");\n    if (mathElement) {\n      const alttext = mathElement.getAttribute(\"alttext\");\n      if (alttext) {\n        return alttext.trim();\n      }\n    }\n\n    const annotation = element.querySelector(\n      'annotation[encoding=\"application/x-tex\"]',\n    );\n    if (annotation?.textContent) {\n      return annotation.textContent.trim();\n    }\n\n    const mathNode = element.nodeName.toLowerCase() === \"math\"\n      ? element\n      : element.querySelector(\"math\");\n    if (mathNode) {\n      return MathMLToLaTeX.convert(mathNode.outerHTML);\n    }\n\n    const imgNode = element.querySelector(\"img\");\n    return imgNode?.getAttribute(\"alt\") || \"\";\n  }\n\n  try {\n    let markdown = turndownService.turndown(processedContent);\n    debugLog(\"Markdown\", \"Markdown conversion successful\");\n\n    // Remove the title from the beginning of the content if it exists\n    const titleMatch = markdown.match(/^# .+\\n+/);\n    if (titleMatch) {\n      markdown = markdown.slice(titleMatch[0].length);\n    }\n\n    // Remove any empty links e.g. [](example.com) that remain, along with surrounding newlines\n    // But don't affect image links like ![](image.jpg)\n    markdown = markdown.replace(/\\n*(? 0) {\n      markdown += \"\\n\\n---\\n\\n\";\n      for (const [id, content] of Object.entries(footnotes)) {\n        markdown += `[^${id}]: ${content}\\n\\n`;\n      }\n    }\n\n    // Clear the footnotes object for the next conversion\n    Object.keys(footnotes).forEach((key) => delete footnotes[key]);\n\n    return markdown.trim();\n  } catch (error) {\n    console.error(\"Error converting HTML to Markdown:\", error);\n    console.log(\n      \"Problematic content:\",\n      processedContent.substring(0, 1000) + \"...\",\n    );\n    return `Partial conversion completed with errors. Original HTML:\\n\\n${processedContent}`;\n  }\n}","import TurndownService from \"npm:turndown\";\nimport { MathMLToLaTeX } from \"npm:mathml-to-latex\";\n// import { processUrls } from \"./string-utils.ts\";\nimport { DOMParser, parseHTML } from \"npm:linkedom\";\n// import { debugLog } from './debug';\n\n// Cases to handle:\n// Full URLs: https://example.com/x.png\n// URLs without protocol: //example.com/x.png\n// Relative URLs:\n// - x.png\n// - /x.png\n// - img/x.png\n// - ../x.png\n\nexport function makeUrlAbsolute(\n  element: Element,\n  attributeName: string,\n  baseUrl: URL,\n) {\n  const attributeValue = element.getAttribute(attributeName);\n  if (attributeValue) {\n    try {\n      // Create a new URL object from the base URL\n      const resolvedBaseUrl = new URL(baseUrl.href);\n\n      // If the base URL points to a file, remove the filename to get the directory\n      if (!resolvedBaseUrl.pathname.endsWith(\"/\")) {\n        resolvedBaseUrl.pathname = resolvedBaseUrl.pathname.substring(\n          0,\n          resolvedBaseUrl.pathname.lastIndexOf(\"/\") + 1,\n        );\n      }\n\n      const url = new URL(attributeValue, resolvedBaseUrl);\n\n      if (![\"http:\", \"https:\"].includes(url.protocol)) {\n        // Handle non-standard protocols (chrome-extension://, moz-extension://, brave://, etc.)\n        const parts = attributeValue.split(\"/\");\n        const firstSegment = parts[2]; // The segment after the protocol\n\n        if (firstSegment && firstSegment.includes(\".\")) {\n          // If it looks like a domain, replace the non-standard protocol with the current page's protocol\n          const newUrl = `${baseUrl.protocol}//` +\n            attributeValue.split(\"://\")[1];\n          element.setAttribute(attributeName, newUrl);\n        } else {\n          // If it doesn't look like a domain it's probably the extension URL, remove the non-standard protocol part and use baseUrl\n          const path = parts.slice(3).join(\"/\");\n          const newUrl =\n            new URL(path, resolvedBaseUrl.origin + resolvedBaseUrl.pathname)\n              .href;\n          element.setAttribute(attributeName, newUrl);\n        }\n      } else {\n        // Handle other cases (relative URLs, protocol-relative URLs)\n        const newUrl = url.href;\n        element.setAttribute(attributeName, newUrl);\n      }\n    } catch (error) {\n      console.warn(`Failed to process URL: ${attributeValue}`, error);\n      element.setAttribute(attributeName, attributeValue);\n    }\n  }\n}\n\n// @todo rewrite to use linkedom DOMParser\nexport function processUrls(htmlContent: string, baseUrl: URL): string {\n  let parser = new DOMParser();\n  let doc = parser.parseFromString(htmlContent, \"text/html\");\n\n  // const tempDiv = document.createElement(\"div\");\n  // tempDiv.innerHTML = htmlContent;\n\n  // Handle relative URLs for both images and links\n  doc.querySelectorAll(\"img\").forEach((img) =>\n    makeUrlAbsolute(img, \"srcset\", baseUrl)\n  );\n  doc.querySelectorAll(\"img\").forEach((img) =>\n    makeUrlAbsolute(img, \"src\", baseUrl)\n  );\n  doc.querySelectorAll(\"a\").forEach((link) =>\n    makeUrlAbsolute(link, \"href\", baseUrl)\n  );\n\n  return doc.toString();\n}\n\nfunction debugLog(...args) {\n  if (import.meta?.env?.NODE_ENV === \"development\") {\n    console.log(...args);\n  }\n}\n\nconst footnotes: { [key: string]: string } = {};\n\nexport function createMarkdownContent(content: string, url: string) {\n  let {\n    HTMLElement,\n    Element,\n    Node,\n    HTMLOListElement,\n    HTMLTableElement,\n    HTMLIFrameElement,\n    HTMLPreElement,\n    HTMLLiElement,\n    HTMLAnchorElement,\n  } = parseHTML(content);\n\n  debugLog(\"Markdown\", \"Starting markdown conversion for URL:\", url);\n  debugLog(\"Markdown\", \"Content length:\", content.length);\n\n  const baseUrl = new URL(url);\n  // Process all URLs at the beginning\n  const processedContent = processUrls(content, baseUrl);\n\n  const turndownService = new TurndownService({\n    headingStyle: \"atx\",\n    hr: \"---\",\n    bulletListMarker: \"-\",\n    codeBlockStyle: \"fenced\",\n    emDelimiter: \"*\",\n    preformattedCode: true,\n  });\n\n  turndownService.addRule(\"table\", {\n    filter: \"table\",\n    replacement: function (content, node) {\n      if (!(node instanceof HTMLTableElement)) return content;\n\n      // Check if it's an ArXiv equation table\n      if (\n        node.classList.contains(\"ltx_equation\") ||\n        node.classList.contains(\"ltx_eqn_table\")\n      ) {\n        return handleNestedEquations(node);\n      }\n\n      // Check if the table has colspan or rowspan\n      const hasComplexStructure = Array.from(node.querySelectorAll(\"td, th\"))\n        .some((cell) =>\n          cell.hasAttribute(\"colspan\") || cell.hasAttribute(\"rowspan\")\n        );\n\n      if (hasComplexStructure) {\n        // Clean up the table HTML\n        const cleanedTable = cleanupTableHTML(node);\n        return \"\\n\\n\" + cleanedTable + \"\\n\\n\";\n      }\n\n      // Process simple tables as before\n      const rows = Array.from(node.rows).map((row) => {\n        const cells = Array.from(row.cells).map((cell) => {\n          // Remove newlines and trim the content\n          let cellContent = turndownService.turndown(cell.innerHTML)\n            .replace(/\\n/g, \" \")\n            .trim();\n          // Escape pipe characters\n          cellContent = cellContent.replace(/\\|/g, \"\\\\|\");\n          return cellContent;\n        });\n        return `| ${cells.join(\" | \")} |`;\n      });\n\n      // Create the separator row\n      const separatorRow = `| ${\n        Array(rows[0].split(\"|\").length - 2).fill(\"---\").join(\" | \")\n      } |`;\n\n      // Combine all rows\n      const tableContent = [rows[0], separatorRow, ...rows.slice(1)].join(\"\\n\");\n\n      return `\\n\\n${tableContent}\\n\\n`;\n    },\n  });\n\n  turndownService.remove([\"style\", \"script\", \"button\"]);\n\n  // Keep iframes, video, audio, sup, and sub elements\n  // @ts-ignore\n  turndownService.keep([\n    \"iframe\",\n    \"video\",\n    \"audio\",\n    \"sup\",\n    \"sub\",\n    \"svg\",\n    \"math\",\n  ]);\n\n  turndownService.addRule(\"list\", {\n    filter: [\"ul\", \"ol\"],\n    replacement: function (content: string, node: Node) {\n      // Remove trailing newlines/spaces from content\n      content = content.trim();\n\n      // Add a newline before the list if it's a top-level list\n      const isTopLevel = !(node.parentNode &&\n        (node.parentNode.nodeName === \"UL\" ||\n          node.parentNode.nodeName === \"OL\"));\n      return (isTopLevel ? \"\\n\" : \"\") + content + \"\\n\";\n    },\n  });\n\n  // Lists with tab indentation\n  turndownService.addRule(\"listItem\", {\n    filter: \"li\",\n    replacement: function (\n      content: string,\n      node: Node,\n      options: TurndownService.Options,\n    ) {\n      if (!(node instanceof HTMLElement) || !(node instanceof HTMLLiElement)) {\n        return content;\n      }\n\n      // Handle task list items\n      const isTaskListItem = node.classList.contains(\"task-list-item\");\n      const checkbox = node.querySelector('input[type=\"checkbox\"]') as\n        | HTMLInputElement\n        | null;\n      let taskListMarker = \"\";\n\n      if (isTaskListItem && checkbox) {\n        // Remove the checkbox from content since we'll add markdown checkbox\n        content = content.replace(/]*>/, \"\");\n        taskListMarker = checkbox.checked ? \"[x] \" : \"[ ] \";\n      }\n\n      content = content\n        // Remove trailing newlines\n        .replace(/\\n+$/, \"\")\n        // Split into lines\n        .split(\"\\n\")\n        // Remove empty lines\n        .filter((line) => line.length > 0)\n        // Add indentation to continued lines\n        .join(\"\\n\\t\");\n\n      let prefix = options.bulletListMarker + \" \";\n      let parent = node.parentNode;\n\n      // Calculate the nesting level\n      let level = 0;\n      let currentParent = node.parentNode;\n      while (\n        currentParent &&\n        (currentParent.nodeName === \"UL\" || currentParent.nodeName === \"OL\")\n      ) {\n        level++;\n        currentParent = currentParent.parentNode;\n      }\n\n      // Add tab indentation based on nesting level, ensuring it's never negative\n      const indentLevel = Math.max(0, level - 1);\n      prefix = \"\\t\".repeat(indentLevel) + prefix;\n\n      if (parent instanceof HTMLOListElement) {\n        let start = parent.getAttribute(\"start\");\n        let index = Array.from(parent.children).indexOf(node as HTMLElement) +\n          1;\n        prefix = \"\\t\".repeat(level - 1) +\n          (start ? Number(start) + index - 1 : index) + \". \";\n      }\n\n      return prefix + taskListMarker + content.trim() +\n        (node.nextSibling && !/\\n$/.test(content) ? \"\\n\" : \"\");\n    },\n  });\n\n  turndownService.addRule(\"figure\", {\n    filter: \"figure\",\n    replacement: function (content, node) {\n      const figure = node as HTMLElement;\n      const img = figure.querySelector(\"img\");\n      const figcaption = figure.querySelector(\"figcaption\");\n\n      if (!img) return content;\n\n      const alt = img.getAttribute(\"alt\") || \"\";\n      let src = img.getAttribute(\"src\") || \"\";\n      const srcset = img.getAttribute(\"srcset\") || \"\";\n\n      if (srcset) {\n        let images: { src: string; width: number }[] = [];\n\n        // deconstruct the srcset urls\n        const srcsetUrls = srcset.split(\",%20\");\n        for (let url of srcsetUrls) {\n          const split = url.split(\"%20\");\n          const srcUrl = split[0];\n          const width = parseInt(split[1]);\n\n          images.push({ src: srcUrl, width });\n        }\n\n        // we need to choose one of the images to use from the srcset\n        // for convenience, simply grab the first one\n        // this could be updated to take width into account\n\n        src = images[0].src;\n      }\n\n      let caption = \"\";\n\n      if (figcaption) {\n        const tagSpan = figcaption.querySelector(\".ltx_tag_figure\");\n        const tagText = tagSpan ? tagSpan.textContent?.trim() : \"\";\n\n        // Process the caption content, including math elements\n        let captionContent = figcaption.innerHTML;\n        captionContent = captionContent.replace(\n          / (.*?)<\\/math>/g,\n          (match, mathContent, offset, string) => {\n            const mathElement = new DOMParser().parseFromString(\n              match,\n              \"text/html\",\n            ).body.firstChild as Element;\n            const latex = extractLatex(mathElement);\n            const prevChar = string[offset - 1] || \"\";\n            const nextChar = string[offset + match.length] || \"\";\n\n            const isStartOfLine = offset === 0 || /\\s/.test(prevChar);\n            const isEndOfLine = offset + match.length === string.length ||\n              /\\s/.test(nextChar);\n\n            const leftSpace = (!isStartOfLine && !/[\\s$]/.test(prevChar))\n              ? \" \"\n              : \"\";\n            const rightSpace = (!isEndOfLine && !/[\\s$]/.test(nextChar))\n              ? \" \"\n              : \"\";\n\n            return `${leftSpace}$${latex}$${rightSpace}`;\n          },\n        );\n\n        // Convert the processed caption content to markdown\n        const captionMarkdown = turndownService.turndown(captionContent);\n\n        // Combine tag and processed caption\n        caption = `${tagText} ${captionMarkdown}`.trim();\n      }\n\n      // Handle references in the caption\n      caption = caption.replace(\n        /\\[([^\\]]+)\\]\\(([^)]+)\\)/g,\n        (match, text, href) => {\n          return `[${text}](${href})`;\n        },\n      );\n\n      return `![${alt}](${src})\\n\\n${caption}\\n\\n`;\n    },\n  });\n\n  // Use Obsidian format for YouTube embeds and tweets\n  turndownService.addRule(\"embedToMarkdown\", {\n    filter: function (node: Node): boolean {\n      if (node instanceof HTMLIFrameElement) {\n        const src = node.getAttribute(\"src\");\n        return !!src && (\n          !!src.match(/(?:youtube\\.com|youtu\\.be)/) ||\n          !!src.match(/(?:twitter\\.com|x\\.com)/)\n        );\n      }\n      return false;\n    },\n    replacement: function (content: string, node: Node): string {\n      if (node instanceof HTMLIFrameElement) {\n        const src = node.getAttribute(\"src\");\n        if (src) {\n          const youtubeMatch = src.match(\n            /(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com|youtu\\.be)\\/(?:embed\\/|watch\\?v=)?([a-zA-Z0-9_-]+)/,\n          );\n          if (youtubeMatch && youtubeMatch[1]) {\n            return `![](https://www.youtube.com/watch?v=${youtubeMatch[1]})`;\n          }\n          const tweetMatch = src.match(\n            /(?:twitter\\.com|x\\.com)\\/.*?(?:status|statuses)\\/(\\d+)/,\n          );\n          if (tweetMatch && tweetMatch[1]) {\n            return `![](https://x.com/i/status/${tweetMatch[1]})`;\n          }\n        }\n      }\n      return content;\n    },\n  });\n\n  turndownService.addRule(\"highlight\", {\n    filter: \"mark\",\n    replacement: function (content) {\n      return \"==\" + content + \"==\";\n    },\n  });\n\n  turndownService.addRule(\"strikethrough\", {\n    filter: (node: Node) =>\n      node.nodeName === \"DEL\" ||\n      node.nodeName === \"S\" ||\n      node.nodeName === \"STRIKE\",\n    replacement: function (content) {\n      return \"~~\" + content + \"~~\";\n    },\n  });\n\n  // Add a new custom rule for complex link structures\n  turndownService.addRule(\"complexLinkStructure\", {\n    filter: function (node, options) {\n      return (\n        node.nodeName === \"A\" &&\n        node.childNodes.length > 1 &&\n        Array.from(node.childNodes).some((child) =>\n          [\"H1\", \"H2\", \"H3\", \"H4\", \"H5\", \"H6\"].includes(child.nodeName)\n        )\n      );\n    },\n    replacement: function (content, node, options) {\n      if (\n        !(node instanceof HTMLElement) || !(node instanceof HTMLAnchorElement)\n      ) return content;\n      const href = node.getAttribute(\"href\");\n      const title = node.getAttribute(\"title\");\n\n      // Extract the heading\n      const headingNode = node.querySelector(\"h1, h2, h3, h4, h5, h6\");\n      const headingContent = headingNode\n        ? turndownService.turndown(headingNode.innerHTML)\n        : \"\";\n\n      // Remove the heading from the content\n      if (headingNode) {\n        headingNode.remove();\n      }\n\n      // Convert the remaining content\n      const remainingContent = turndownService.turndown(node.innerHTML);\n\n      // Construct the new markdown\n      let markdown = `${headingContent}\\n\\n${remainingContent}\\n\\n`;\n      if (href) {\n        markdown += `[View original](${href})`;\n        if (title) {\n          markdown += ` \"${title}\"`;\n        }\n      }\n\n      return markdown;\n    },\n  });\n\n  turndownService.addRule(\"arXivEnumerate\", {\n    filter: (node) => {\n      return node.nodeName === \"OL\" && node.classList.contains(\"ltx_enumerate\");\n    },\n    replacement: function (content, node) {\n      if (\n        !(node instanceof HTMLElement) || !(node instanceof HTMLOListElement)\n      ) return content;\n\n      const items = Array.from(node.children).map((item, index) => {\n        if (item instanceof HTMLElement) {\n          const itemContent = item.innerHTML.replace(\n            /^

\\d+\\.<\\/span>\\s*/,\n \"\",\n );\n return `${index + 1}. ${turndownService.turndown(itemContent)}`;\n }\n return \"\";\n });\n\n return \"\\n\\n\" + items.join(\"\\n\\n\") + \"\\n\\n\";\n },\n });\n\n turndownService.addRule(\"removeHiddenElements\", {\n filter: function (node) {\n return (\n node.style.display === \"none\"\n );\n },\n replacement: function () {\n return \"\";\n },\n });\n\n turndownService.addRule(\"citations\", {\n filter: (node: Node): boolean => {\n if (node instanceof Element) {\n return (\n node.nodeName === \"SUP\" && node.id.startsWith(\"fnref:\")\n );\n }\n return false;\n },\n replacement: (content, node) => {\n if (node instanceof HTMLElement) {\n if (node.nodeName === \"SUP\" && node.id.startsWith(\"fnref:\")) {\n const id = node.id.replace(\"fnref:\", \"\");\n // Extract only the primary number before any hyphen\n const primaryNumber = id.split(\"-\")[0];\n return `[^${primaryNumber}]`;\n }\n }\n return content;\n },\n });\n\n // Footnotes list\n turndownService.addRule(\"footnotesList\", {\n filter: (node: Node): boolean => {\n if (node instanceof HTMLOListElement) {\n return (\n node.parentElement?.id === \"footnotes\"\n );\n }\n return false;\n },\n replacement: (content, node) => {\n if ((node instanceof HTMLElement) || (node instanceof HTMLOListElement)) {\n const references = Array.from(node.children).map((li) => {\n let id;\n if (li.id.startsWith(\"fn:\")) {\n id = li.id.replace(\"fn:\", \"\");\n } else {\n const match = li.id.split(\"/\").pop()?.match(/cite_note-(.+)/);\n id = match ? match[1] : li.id;\n }\n\n // Remove the leading sup element if its content matches the footnote id\n const supElement = li.querySelector(\"sup\");\n if (supElement && supElement.textContent?.trim() === id) {\n supElement.remove();\n }\n\n const referenceContent = turndownService.turndown(li.innerHTML);\n // Remove the backlink from the footnote content\n const cleanedContent = referenceContent.replace(/\\s*↩︎$/, \"\").trim();\n return `[^${id.toLowerCase()}]: ${cleanedContent}`;\n });\n return \"\\n\\n\" + references.join(\"\\n\\n\") + \"\\n\\n\";\n }\n return content;\n },\n });\n\n // General removal rules for varous website elements\n turndownService.addRule(\"removals\", {\n filter: function (node) {\n if (!(node instanceof HTMLElement)) return false;\n // Remove the Defuddle backlink from the footnote content\n if (node.getAttribute(\"href\")?.includes(\"#fnref\")) return true;\n if (node.classList.contains(\"footnote-backref\")) return true;\n return false;\n },\n replacement: function (content, node) {\n return \"\";\n },\n });\n\n turndownService.addRule(\"handleTextNodesInTables\", {\n filter: function (node: Node): boolean {\n return node.nodeType === Node.TEXT_NODE &&\n node.parentNode !== null &&\n node.parentNode.nodeName === \"TD\";\n },\n replacement: function (content: string): string {\n return content;\n },\n });\n\n turndownService.addRule(\"preformattedCode\", {\n filter: (node) => {\n return node.nodeName === \"PRE\";\n },\n replacement: (content, node) => {\n // if (!(node instanceof HTMLElement) || !(node instanceof HTMLPreElement)) {\n // console.log(node);\n // return content;\n // }\n\n const codeElement = node.querySelector(\"code\");\n if (!codeElement) {\n return content;\n }\n\n const language = codeElement.getAttribute(\"data-lang\") || \"\";\n const code = codeElement.textContent || \"\";\n\n // Clean up the content and escape backticks\n const cleanCode = code\n .trim()\n .replace(/`/g, \"\\\\`\");\n\n return `\\n\\`\\`\\`${language}\\n${cleanCode}\\n\\`\\`\\`\\n`;\n },\n });\n\n turndownService.addRule(\"MathJax\", {\n filter: (node) => {\n const isMjxContainer = node.nodeName.toLowerCase() === \"mjx-container\";\n return isMjxContainer;\n },\n replacement: (content, node) => {\n if (!(node instanceof HTMLElement)) {\n return content;\n }\n\n const assistiveMml = node.querySelector(\"mjx-assistive-mml\");\n if (!assistiveMml) {\n return content;\n }\n\n const mathElement = assistiveMml.querySelector(\"math\");\n if (!mathElement) {\n return content;\n }\n\n let latex;\n try {\n latex = MathMLToLaTeX.convert(mathElement.outerHTML);\n } catch (error) {\n console.error(\"Error converting MathML to LaTeX:\", error);\n return content;\n }\n\n // Check if it's an inline or block math element\n const isBlock = mathElement.getAttribute(\"display\") === \"block\";\n\n if (isBlock) {\n return `\\n$$\\n${latex}\\n$$\\n`;\n } else {\n return `$${latex}$`;\n }\n },\n });\n\n turndownService.addRule(\"math\", {\n filter: (node) => {\n return node.nodeName.toLowerCase() === \"math\" ||\n (node instanceof Element && node.classList &&\n (node.classList.contains(\"mwe-math-element\") ||\n node.classList.contains(\"mwe-math-fallback-image-inline\") ||\n node.classList.contains(\"mwe-math-fallback-image-display\")));\n },\n replacement: (content, node) => {\n if (!(node instanceof Element)) return content;\n\n let latex = extractLatex(node);\n\n // Remove leading and trailing whitespace\n latex = latex.trim();\n\n // Check if the math element is within a table\n const isInTable = node.closest(\"table\") !== null;\n\n // Check if it's an inline or block math element\n if (\n !isInTable && (\n node.getAttribute(\"display\") === \"block\" ||\n node.classList.contains(\"mwe-math-fallback-image-display\") ||\n (node.parentElement &&\n node.parentElement.classList.contains(\"mwe-math-element\") &&\n node.parentElement.previousElementSibling &&\n node.parentElement.previousElementSibling.nodeName.toLowerCase() ===\n \"p\")\n )\n ) {\n return `\\n$$\\n${latex}\\n$$\\n`;\n } else {\n // For inline math, ensure there's a space before and after only if needed\n const prevNode = node.previousSibling;\n const nextNode = node.nextSibling;\n const prevChar = prevNode?.textContent?.slice(-1) || \"\";\n const nextChar = nextNode?.textContent?.[0] || \"\";\n\n const isStartOfLine = !prevNode ||\n (prevNode.nodeType === Node.TEXT_NODE &&\n prevNode.textContent?.trim() === \"\");\n const isEndOfLine = !nextNode ||\n (nextNode.nodeType === Node.TEXT_NODE &&\n nextNode.textContent?.trim() === \"\");\n\n const leftSpace =\n (!isStartOfLine && prevChar && !/[\\s$]/.test(prevChar)) ? \" \" : \"\";\n const rightSpace = (!isEndOfLine && nextChar && !/[\\s$]/.test(nextChar))\n ? \" \"\n : \"\";\n\n return `${leftSpace}$${latex}$${rightSpace}`;\n }\n },\n });\n\n turndownService.addRule(\"katex\", {\n filter: (node) => {\n return node instanceof HTMLElement &&\n (node.classList.contains(\"math\") || node.classList.contains(\"katex\"));\n },\n replacement: (content, node) => {\n if (!(node instanceof HTMLElement)) return content;\n\n // Try to find the original LaTeX content\n // 1. Check data-latex attribute\n let latex = node.getAttribute(\"data-latex\");\n\n // 2. If no data-latex, try to get from .katex-mathml\n if (!latex) {\n const mathml = node.querySelector(\n '.katex-mathml annotation[encoding=\"application/x-tex\"]',\n );\n latex = mathml?.textContent || \"\";\n }\n\n // 3. If still no content, use text content as fallback\n if (!latex) {\n latex = node.textContent?.trim() || \"\";\n }\n\n // Determine if it's an inline formula\n const mathElement = node.querySelector(\".katex-mathml math\");\n const isInline = node.classList.contains(\"math-inline\") ||\n (mathElement && mathElement.getAttribute(\"display\") !== \"block\");\n\n if (isInline) {\n return `$${latex}$`;\n } else {\n return `\\n$$\\n${latex}\\n$$\\n`;\n }\n },\n });\n\n turndownService.addRule(\"callout\", {\n filter: (node) => {\n return (\n node.nodeName.toLowerCase() === \"div\" &&\n node.classList.contains(\"markdown-alert\")\n );\n },\n replacement: (content, node) => {\n const element = node as HTMLElement;\n\n // Get alert type from the class (e.g., markdown-alert-note -> NOTE)\n const alertClasses = Array.from(element.classList);\n const typeClass = alertClasses.find((c) =>\n c.startsWith(\"markdown-alert-\") && c !== \"markdown-alert\"\n );\n const type = typeClass\n ? typeClass.replace(\"markdown-alert-\", \"\").toUpperCase()\n : \"NOTE\";\n\n // Find the title element and content\n const titleElement = element.querySelector(\".markdown-alert-title\");\n const contentElement = element.querySelector(\n \"p:not(.markdown-alert-title)\",\n );\n\n // Extract content, removing the title from it if present\n let alertContent = content;\n if (titleElement && titleElement.textContent) {\n alertContent = contentElement?.textContent ||\n content.replace(titleElement.textContent, \"\");\n }\n\n // Format as Obsidian callout\n return `\\n> [!${type}]\\n> ${\n alertContent.trim().replace(/\\n/g, \"\\n> \")\n }\\n`;\n },\n });\n\n function handleNestedEquations(table: Element): string {\n const mathElements = table.querySelectorAll(\"math[alttext]\");\n if (mathElements.length === 0) return \"\";\n\n return Array.from(mathElements).map((mathElement) => {\n const alttext = mathElement.getAttribute(\"alttext\");\n if (alttext) {\n // Check if it's an inline or block equation\n const isInline = mathElement.closest(\".ltx_eqn_inline\") !== null;\n return isInline ? `$${alttext.trim()}$` : `\\n$$\\n${alttext.trim()}\\n$$`;\n }\n return \"\";\n }).join(\"\\n\\n\");\n }\n\n function cleanupTableHTML(table: HTMLTableElement): string {\n const allowedAttributes = [\n \"src\",\n \"href\",\n \"style\",\n \"align\",\n \"width\",\n \"height\",\n \"rowspan\",\n \"colspan\",\n \"bgcolor\",\n \"scope\",\n \"valign\",\n \"headers\",\n ];\n\n const cleanElement = (element: Element) => {\n Array.from(element.attributes).forEach((attr) => {\n if (!allowedAttributes.includes(attr.name)) {\n element.removeAttribute(attr.name);\n }\n });\n\n element.childNodes.forEach((child) => {\n if (child instanceof Element) {\n cleanElement(child);\n }\n });\n };\n\n // Create a clone of the table to avoid modifying the original DOM\n const tableClone = table.cloneNode(true) as HTMLTableElement;\n cleanElement(tableClone);\n\n return tableClone.outerHTML;\n }\n\n function extractLatex(element: Element): string {\n // Check if the element is a

element and has an alttext attribute\n    if (element.nodeName.toLowerCase() === \"math\") {\n      let latex = element.getAttribute(\"data-latex\");\n      let alttext = element.getAttribute(\"alttext\");\n      if (latex) {\n        return latex.trim();\n      } else if (alttext) {\n        return alttext.trim();\n      }\n      console.log(\"No latex or alttext found for math element:\", element);\n    }\n\n    // If not, look for a nested element with alttext\n    const mathElement = element.querySelector(\"math[alttext]\");\n    if (mathElement) {\n      const alttext = mathElement.getAttribute(\"alttext\");\n      if (alttext) {\n        return alttext.trim();\n      }\n    }\n\n    const annotation = element.querySelector(\n      'annotation[encoding=\"application/x-tex\"]',\n    );\n    if (annotation?.textContent) {\n      return annotation.textContent.trim();\n    }\n\n    const mathNode = element.nodeName.toLowerCase() === \"math\"\n      ? element\n      : element.querySelector(\"math\");\n    if (mathNode) {\n      return MathMLToLaTeX.convert(mathNode.outerHTML);\n    }\n\n    const imgNode = element.querySelector(\"img\");\n    return imgNode?.getAttribute(\"alt\") || \"\";\n  }\n\n  try {\n    let markdown = turndownService.turndown(processedContent);\n    debugLog(\"Markdown\", \"Markdown conversion successful\");\n\n    // Remove the title from the beginning of the content if it exists\n    const titleMatch = markdown.match(/^# .+\\n+/);\n    if (titleMatch) {\n      markdown = markdown.slice(titleMatch[0].length);\n    }\n\n    // Remove any empty links e.g. [](example.com) that remain, along with surrounding newlines\n    // But don't affect image links like ![](image.jpg)\n    markdown = markdown.replace(/\\n*(? 0) {\n      markdown += \"\\n\\n---\\n\\n\";\n      for (const [id, content] of Object.entries(footnotes)) {\n        markdown += `[^${id}]: ${content}\\n\\n`;\n      }\n    }\n\n    // Clear the footnotes object for the next conversion\n    Object.keys(footnotes).forEach((key) => delete footnotes[key]);\n\n    return markdown.trim();\n  } catch (error) {\n    console.error(\"Error converting HTML to Markdown:\", error);\n    console.log(\n      \"Problematic content:\",\n      processedContent.substring(0, 1000) + \"...\",\n    );\n    return `Partial conversion completed with errors. Original HTML:\\n\\n${processedContent}`;\n  }\n}","createdAt",["D",1758282055828],"branchVersion","branchId","36e1d838-8d0c-11f0-873c-0224a6c84d84","id","36e184a0-8d0c-11f0-873c-0224a6c84d84","html-to-markdown-experiments",["D",1757374404832],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","2ac99154-7d90-4864-83c7-90f839921aea","pinnedFileId","36e411d4-8d0c-11f0-873c-0224a6c84d84","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","a2b140c3-8bfe-4d69-9408-83fa547cb071","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_96":95,"_103":99},"org",{"_96":95,"_97":-5,"_98":99,"_18":54,"_100":101,"_102":25},"branch",{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"isOwned",75,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","matthamlin","handle","displayName","avatar","https://secure.gravatar.com/avatar/77c5c43c3a9cd2808c13784289b34d52&d=identicon","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":175,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":56,"_57":58,"_59":58,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":176,"_80":177,"_82":178,"_84":25},"tree",[162,163,164],"files",[162,163,164],"counts",{"_151":62,"_152":153,"_154":62,"_155":156,"_157":158},"contents",[126,127,128],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":147,"_32":28,"_30":31,"_132":-5,"_41":42,"_40":14,"_134":58,"_38":148,"_13":149,"_33":34,"_35":37,"_24":25,"_138":-5,"_27":150},{"_43":140,"_32":141,"_30":56,"_132":-5,"_41":42,"_40":142,"_134":25,"_38":143,"_13":144,"_33":34,"_35":145,"_24":25,"_138":-5,"_27":146},{"_43":129,"_32":130,"_30":131,"_132":-5,"_41":42,"_40":133,"_134":58,"_38":135,"_13":136,"_33":34,"_35":137,"_24":25,"_138":-5,"_27":139},"027f31ea-8d10-11f0-a45f-0224a6c84d84","string-utils.ts","78aa8706-8d0c-11f0-a9be-0224a6c84d84","parentId",19,"latest",["D",1757376034929],10,"// deno-lint-ignore-file prefer-const\nimport { DOMParser } from \"npm:linkedom\";\n\n// Cases to handle:\n// Full URLs: https://example.com/x.png\n// URLs without protocol: //example.com/x.png\n// Relative URLs:\n// - x.png\n// - /x.png\n// - img/x.png\n// - ../x.png\n\nexport function makeUrlAbsolute(\n  element: Element,\n  attributeName: string,\n  baseUrl: URL,\n) {\n  const attributeValue = element.getAttribute(attributeName);\n  if (attributeValue) {\n    try {\n      // Create a new URL object from the base URL\n      const resolvedBaseUrl = new URL(baseUrl.href);\n\n      // If the base URL points to a file, remove the filename to get the directory\n      if (!resolvedBaseUrl.pathname.endsWith(\"/\")) {\n        resolvedBaseUrl.pathname = resolvedBaseUrl.pathname.substring(\n          0,\n          resolvedBaseUrl.pathname.lastIndexOf(\"/\") + 1,\n        );\n      }\n\n      const url = new URL(attributeValue, resolvedBaseUrl);\n\n      if (![\"http:\", \"https:\"].includes(url.protocol)) {\n        // Handle non-standard protocols (chrome-extension://, moz-extension://, brave://, etc.)\n        const parts = attributeValue.split(\"/\");\n        const firstSegment = parts[2]; // The segment after the protocol\n\n        if (firstSegment && firstSegment.includes(\".\")) {\n          // If it looks like a domain, replace the non-standard protocol with the current page's protocol\n          const newUrl = `${baseUrl.protocol}//` +\n            attributeValue.split(\"://\")[1];\n          element.setAttribute(attributeName, newUrl);\n        } else {\n          // If it doesn't look like a domain it's probably the extension URL, remove the non-standard protocol part and use baseUrl\n          const path = parts.slice(3).join(\"/\");\n          const newUrl =\n            new URL(path, resolvedBaseUrl.origin + resolvedBaseUrl.pathname)\n              .href;\n          element.setAttribute(attributeName, newUrl);\n        }\n      } else {\n        // Handle other cases (relative URLs, protocol-relative URLs)\n        const newUrl = url.href;\n        element.setAttribute(attributeName, newUrl);\n      }\n    } catch (error) {\n      console.warn(`Failed to process URL: ${attributeValue}`, error);\n      element.setAttribute(attributeName, attributeValue);\n    }\n  }\n}\n\n// @todo rewrite to use linkedom DOMParser\nexport function processUrls(htmlContent: string, baseUrl: URL): string {\n  let parser = new DOMParser();\n  let doc = parser.parseFromString(htmlContent, \"text/html\");\n\n  // const tempDiv = document.createElement(\"div\");\n  // tempDiv.innerHTML = htmlContent;\n\n  // Handle relative URLs for both images and links\n  doc.querySelectorAll(\"img\").forEach((img) =>\n    makeUrlAbsolute(img, \"srcset\", baseUrl)\n  );\n  doc.querySelectorAll(\"img\").forEach((img) =>\n    makeUrlAbsolute(img, \"src\", baseUrl)\n  );\n  doc.querySelectorAll(\"a\").forEach((link) =>\n    makeUrlAbsolute(link, \"href\", baseUrl)\n  );\n\n  return doc.toString();\n}","label",[130],"f20e4d4e-8d11-11f0-9eda-0224a6c84d84","main.ts",30,["D",1757376866334],8,"import { createMarkdownContent } from \"./html-to-markdown.ts\";\n\nlet htmlStr =\n  `

Testing

\"use server-entry\";

export async function Comments() {
  // Load data from a database...
  let comments = await db.getComments();

  return comments.map((comment) => (
    <article key={comment.id}>
      <p>Posted by: {comment.user}</p>
      {renderMarkdown(comment.body)}
    </article>
  ));
}

`;\nlet res = createMarkdownContent(htmlStr, \"http://localhost\");\n\nconsole.log(res);",[141],"8109fd0c-954d-11f0-a279-0224a6c84d84",["D",1758282055828],23,[28],"pulls","branches",1,"forks","envVars",{"_159":62,"_161":62},"connections",{"_159":62,"_160":62},"count","unfilledCount","emptyCount",{"_43":147,"_32":28,"_30":31,"_132":-5,"_41":42,"_40":14,"_134":58,"_38":172,"_13":149,"_33":34,"_35":37,"_24":25,"_138":-5,"_166":173,"_27":174},{"_43":140,"_32":141,"_30":56,"_132":-5,"_41":42,"_40":142,"_134":25,"_38":169,"_13":144,"_33":34,"_35":145,"_24":25,"_138":-5,"_166":170,"_27":171},{"_43":129,"_32":130,"_30":131,"_132":-5,"_41":42,"_40":133,"_134":58,"_38":165,"_13":136,"_33":34,"_35":137,"_24":25,"_138":-5,"_166":167,"_27":168},["D",1757376034929],"children",[],[130],["D",1757376866334],[],[141],["D",1758282055828],[],[28],["D",1757374404832],{"_96":95,"_103":99},{"_96":95,"_97":-5,"_98":99,"_18":54,"_100":101,"_102":25},{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"ctx",{"_78":-5,"_123":212,"_213":214,"_215":216,"_217":25,"_218":25,"_219":25,"_220":58,"_221":-5,"_222":-7,"_223":-7,"_224":62,"_225":25,"_226":25,"_227":-5,"_228":25,"_229":25,"_230":25,"_231":-5},"ENV",{"_183":184,"_185":186,"_187":86,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197,"_198":199,"_200":201,"_202":203,"_204":205,"_206":207,"_208":209,"_210":211},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","fdb67791413f4386b4dbf18f354029a617f6160f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v44