element and has an alttext attribute\n    if (element.nodeName.toLowerCase() === \"math\") {\n      let latex = element.getAttribute(\"data-latex\");\n      let alttext = element.getAttribute(\"alttext\");\n      if (latex) {\n        return latex.trim();\n      } else if (alttext) {\n        return alttext.trim();\n      }\n      console.log(\"No latex or alttext found for math element:\", element);\n    }\n\n    // If not, look for a nested element with alttext\n    const mathElement = element.querySelector(\"math[alttext]\");\n    if (mathElement) {\n      const alttext = mathElement.getAttribute(\"alttext\");\n      if (alttext) {\n        return alttext.trim();\n      }\n    }\n\n    const annotation = element.querySelector(\n      'annotation[encoding=\"application/x-tex\"]',\n    );\n    if (annotation?.textContent) {\n      return annotation.textContent.trim();\n    }\n\n    const mathNode = element.nodeName.toLowerCase() === \"math\"\n      ? element\n      : element.querySelector(\"math\");\n    if (mathNode) {\n      return MathMLToLaTeX.convert(mathNode.outerHTML);\n    }\n\n    const imgNode = element.querySelector(\"img\");\n    return imgNode?.getAttribute(\"alt\") || \"\";\n  }\n\n  try {\n    let markdown = turndownService.turndown(processedContent);\n    debugLog(\"Markdown\", \"Markdown conversion successful\");\n\n    // Remove the title from the beginning of the content if it exists\n    const titleMatch = markdown.match(/^# .+\\n+/);\n    if (titleMatch) {\n      markdown = markdown.slice(titleMatch[0].length);\n    }\n\n    // Remove any empty links e.g. [](example.com) that remain, along with surrounding newlines\n    // But don't affect image links like ![](image.jpg)\n    markdown = markdown.replace(/\\n*(? 0) {\n      markdown += \"\\n\\n---\\n\\n\";\n      for (const [id, content] of Object.entries(footnotes)) {\n        markdown += `[^${id}]: ${content}\\n\\n`;\n      }\n    }\n\n    // Clear the footnotes object for the next conversion\n    Object.keys(footnotes).forEach((key) => delete footnotes[key]);\n\n    return markdown.trim();\n  } catch (error) {\n    console.error(\"Error converting HTML to Markdown:\", error);\n    console.log(\n      \"Problematic content:\",\n      processedContent.substring(0, 1000) + \"...\",\n    );\n    return `Partial conversion completed with errors. Original HTML:\\n\\n${processedContent}`;\n  }\n}","deleted","label","children",[],[150],{"_102":171,"_95":172,"_151":173,"_153":-5,"_154":136,"_155":135,"_157":158,"_104":174,"_99":175,"_161":176,"_163":177,"_165":22,"_166":-5,"_167":178,"_98":179},"8774660e-956c-11f0-94d5-0224a6c84d84","main.ts","36e411d4-8d0c-11f0-873c-0224a6c84d84",["D",1758295380994],39,"http","// import { createMarkdownContent } from \"./html-to-markdown.ts\";\n\nimport TurndownService from \"npm:turndown\";\nimport { MathMLToLaTeX } from \"npm:mathml-to-latex\";\n// import { processUrls } from \"./string-utils.ts\";\nimport { DOMParser, parseHTML } from \"npm:linkedom\";\n// import { debugLog } from './debug';\n\n// Cases to handle:\n// Full URLs: https://example.com/x.png\n// URLs without protocol: //example.com/x.png\n// Relative URLs:\n// - x.png\n// - /x.png\n// - img/x.png\n// - ../x.png\n\nexport function makeUrlAbsolute(\n  element: Element,\n  attributeName: string,\n  baseUrl: URL,\n) {\n  const attributeValue = element.getAttribute(attributeName);\n  if (attributeValue) {\n    try {\n      // Create a new URL object from the base URL\n      const resolvedBaseUrl = new URL(baseUrl.href);\n\n      // If the base URL points to a file, remove the filename to get the directory\n      if (!resolvedBaseUrl.pathname.endsWith(\"/\")) {\n        resolvedBaseUrl.pathname = resolvedBaseUrl.pathname.substring(\n          0,\n          resolvedBaseUrl.pathname.lastIndexOf(\"/\") + 1,\n        );\n      }\n\n      const url = new URL(attributeValue, resolvedBaseUrl);\n\n      if (![\"http:\", \"https:\"].includes(url.protocol)) {\n        // Handle non-standard protocols (chrome-extension://, moz-extension://, brave://, etc.)\n        const parts = attributeValue.split(\"/\");\n        const firstSegment = parts[2]; // The segment after the protocol\n\n        if (firstSegment && firstSegment.includes(\".\")) {\n          // If it looks like a domain, replace the non-standard protocol with the current page's protocol\n          const newUrl = `${baseUrl.protocol}//` +\n            attributeValue.split(\"://\")[1];\n          element.setAttribute(attributeName, newUrl);\n        } else {\n          // If it doesn't look like a domain it's probably the extension URL, remove the non-standard protocol part and use baseUrl\n          const path = parts.slice(3).join(\"/\");\n          const newUrl =\n            new URL(path, resolvedBaseUrl.origin + resolvedBaseUrl.pathname)\n              .href;\n          element.setAttribute(attributeName, newUrl);\n        }\n      } else {\n        // Handle other cases (relative URLs, protocol-relative URLs)\n        const newUrl = url.href;\n        element.setAttribute(attributeName, newUrl);\n      }\n    } catch (error) {\n      console.warn(`Failed to process URL: ${attributeValue}`, error);\n      element.setAttribute(attributeName, attributeValue);\n    }\n  }\n}\n\n// @todo rewrite to use linkedom DOMParser\nexport function processUrls(htmlContent: string, baseUrl: URL): string {\n  let parser = new DOMParser();\n  let doc = parser.parseFromString(htmlContent, \"text/html\");\n\n  // const tempDiv = document.createElement(\"div\");\n  // tempDiv.innerHTML = htmlContent;\n\n  // Handle relative URLs for both images and links\n  doc.querySelectorAll(\"img\").forEach((img) =>\n    makeUrlAbsolute(img, \"srcset\", baseUrl)\n  );\n  doc.querySelectorAll(\"img\").forEach((img) =>\n    makeUrlAbsolute(img, \"src\", baseUrl)\n  );\n  doc.querySelectorAll(\"a\").forEach((link) =>\n    makeUrlAbsolute(link, \"href\", baseUrl)\n  );\n\n  return doc.toString();\n}\n\nfunction debugLog(...args) {\n  if (import.meta?.env?.NODE_ENV === \"development\") {\n    console.log(...args);\n  }\n}\n\nconst footnotes: { [key: string]: string } = {};\n\nexport function createMarkdownContent(content: string, url: string) {\n  let {\n    HTMLElement,\n    Element,\n    Node,\n    HTMLOListElement,\n    HTMLTableElement,\n    HTMLIFrameElement,\n    HTMLPreElement,\n    HTMLLiElement,\n    HTMLAnchorElement,\n  } = parseHTML(content);\n\n  debugLog(\"Markdown\", \"Starting markdown conversion for URL:\", url);\n  debugLog(\"Markdown\", \"Content length:\", content.length);\n\n  const baseUrl = new URL(url);\n  // Process all URLs at the beginning\n  const processedContent = processUrls(content, baseUrl);\n\n  const turndownService = new TurndownService({\n    headingStyle: \"atx\",\n    hr: \"---\",\n    bulletListMarker: \"-\",\n    codeBlockStyle: \"fenced\",\n    emDelimiter: \"*\",\n    preformattedCode: true,\n  });\n\n  turndownService.addRule(\"table\", {\n    filter: \"table\",\n    replacement: function (content, node) {\n      if (!(node instanceof HTMLTableElement)) return content;\n\n      // Check if it's an ArXiv equation table\n      if (\n        node.classList.contains(\"ltx_equation\") ||\n        node.classList.contains(\"ltx_eqn_table\")\n      ) {\n        return handleNestedEquations(node);\n      }\n\n      // Check if the table has colspan or rowspan\n      const hasComplexStructure = Array.from(node.querySelectorAll(\"td, th\"))\n        .some((cell) =>\n          cell.hasAttribute(\"colspan\") || cell.hasAttribute(\"rowspan\")\n        );\n\n      if (hasComplexStructure) {\n        // Clean up the table HTML\n        const cleanedTable = cleanupTableHTML(node);\n        return \"\\n\\n\" + cleanedTable + \"\\n\\n\";\n      }\n\n      // Process simple tables as before\n      const rows = Array.from(node.rows).map((row) => {\n        const cells = Array.from(row.cells).map((cell) => {\n          // Remove newlines and trim the content\n          let cellContent = turndownService.turndown(cell.innerHTML)\n            .replace(/\\n/g, \" \")\n            .trim();\n          // Escape pipe characters\n          cellContent = cellContent.replace(/\\|/g, \"\\\\|\");\n          return cellContent;\n        });\n        return `| ${cells.join(\" | \")} |`;\n      });\n\n      // Create the separator row\n      const separatorRow = `| ${\n        Array(rows[0].split(\"|\").length - 2).fill(\"---\").join(\" | \")\n      } |`;\n\n      // Combine all rows\n      const tableContent = [rows[0], separatorRow, ...rows.slice(1)].join(\"\\n\");\n\n      return `\\n\\n${tableContent}\\n\\n`;\n    },\n  });\n\n  turndownService.remove([\"style\", \"script\", \"button\"]);\n\n  // Keep iframes, video, audio, sup, and sub elements\n  // @ts-ignore\n  turndownService.keep([\n    \"iframe\",\n    \"video\",\n    \"audio\",\n    \"sup\",\n    \"sub\",\n    \"svg\",\n    \"math\",\n  ]);\n\n  turndownService.addRule(\"list\", {\n    filter: [\"ul\", \"ol\"],\n    replacement: function (content: string, node: Node) {\n      // Remove trailing newlines/spaces from content\n      content = content.trim();\n\n      // Add a newline before the list if it's a top-level list\n      const isTopLevel = !(node.parentNode &&\n        (node.parentNode.nodeName === \"UL\" ||\n          node.parentNode.nodeName === \"OL\"));\n      return (isTopLevel ? \"\\n\" : \"\") + content + \"\\n\";\n    },\n  });\n\n  // Lists with tab indentation\n  turndownService.addRule(\"listItem\", {\n    filter: \"li\",\n    replacement: function (\n      content: string,\n      node: Node,\n      options: TurndownService.Options,\n    ) {\n      if (!(node instanceof HTMLElement) || !(node instanceof HTMLLiElement)) {\n        return content;\n      }\n\n      // Handle task list items\n      const isTaskListItem = node.classList.contains(\"task-list-item\");\n      const checkbox = node.querySelector('input[type=\"checkbox\"]') as\n        | HTMLInputElement\n        | null;\n      let taskListMarker = \"\";\n\n      if (isTaskListItem && checkbox) {\n        // Remove the checkbox from content since we'll add markdown checkbox\n        content = content.replace(/]*>/, \"\");\n        taskListMarker = checkbox.checked ? \"[x] \" : \"[ ] \";\n      }\n\n      content = content\n        // Remove trailing newlines\n        .replace(/\\n+$/, \"\")\n        // Split into lines\n        .split(\"\\n\")\n        // Remove empty lines\n        .filter((line) => line.length > 0)\n        // Add indentation to continued lines\n        .join(\"\\n\\t\");\n\n      let prefix = options.bulletListMarker + \" \";\n      let parent = node.parentNode;\n\n      // Calculate the nesting level\n      let level = 0;\n      let currentParent = node.parentNode;\n      while (\n        currentParent &&\n        (currentParent.nodeName === \"UL\" || currentParent.nodeName === \"OL\")\n      ) {\n        level++;\n        currentParent = currentParent.parentNode;\n      }\n\n      // Add tab indentation based on nesting level, ensuring it's never negative\n      const indentLevel = Math.max(0, level - 1);\n      prefix = \"\\t\".repeat(indentLevel) + prefix;\n\n      if (parent instanceof HTMLOListElement) {\n        let start = parent.getAttribute(\"start\");\n        let index = Array.from(parent.children).indexOf(node as HTMLElement) +\n          1;\n        prefix = \"\\t\".repeat(level - 1) +\n          (start ? Number(start) + index - 1 : index) + \". \";\n      }\n\n      return prefix + taskListMarker + content.trim() +\n        (node.nextSibling && !/\\n$/.test(content) ? \"\\n\" : \"\");\n    },\n  });\n\n  turndownService.addRule(\"figure\", {\n    filter: \"figure\",\n    replacement: function (content, node) {\n      const figure = node as HTMLElement;\n      const img = figure.querySelector(\"img\");\n      const figcaption = figure.querySelector(\"figcaption\");\n\n      if (!img) return content;\n\n      const alt = img.getAttribute(\"alt\") || \"\";\n      let src = img.getAttribute(\"src\") || \"\";\n      const srcset = img.getAttribute(\"srcset\") || \"\";\n\n      if (srcset) {\n        let images: { src: string; width: number }[] = [];\n\n        // deconstruct the srcset urls\n        const srcsetUrls = srcset.split(\",%20\");\n        for (let url of srcsetUrls) {\n          const split = url.split(\"%20\");\n          const srcUrl = split[0];\n          const width = parseInt(split[1]);\n\n          images.push({ src: srcUrl, width });\n        }\n\n        // we need to choose one of the images to use from the srcset\n        // for convenience, simply grab the first one\n        // this could be updated to take width into account\n\n        src = images[0].src;\n      }\n\n      let caption = \"\";\n\n      if (figcaption) {\n        const tagSpan = figcaption.querySelector(\".ltx_tag_figure\");\n        const tagText = tagSpan ? tagSpan.textContent?.trim() : \"\";\n\n        // Process the caption content, including math elements\n        let captionContent = figcaption.innerHTML;\n        captionContent = captionContent.replace(\n          / (.*?)<\\/math>/g,\n          (match, mathContent, offset, string) => {\n            const mathElement = new DOMParser().parseFromString(\n              match,\n              \"text/html\",\n            ).body.firstChild as Element;\n            const latex = extractLatex(mathElement);\n            const prevChar = string[offset - 1] || \"\";\n            const nextChar = string[offset + match.length] || \"\";\n\n            const isStartOfLine = offset === 0 || /\\s/.test(prevChar);\n            const isEndOfLine = offset + match.length === string.length ||\n              /\\s/.test(nextChar);\n\n            const leftSpace = (!isStartOfLine && !/[\\s$]/.test(prevChar))\n              ? \" \"\n              : \"\";\n            const rightSpace = (!isEndOfLine && !/[\\s$]/.test(nextChar))\n              ? \" \"\n              : \"\";\n\n            return `${leftSpace}$${latex}$${rightSpace}`;\n          },\n        );\n\n        // Convert the processed caption content to markdown\n        const captionMarkdown = turndownService.turndown(captionContent);\n\n        // Combine tag and processed caption\n        caption = `${tagText} ${captionMarkdown}`.trim();\n      }\n\n      // Handle references in the caption\n      caption = caption.replace(\n        /\\[([^\\]]+)\\]\\(([^)]+)\\)/g,\n        (match, text, href) => {\n          return `[${text}](${href})`;\n        },\n      );\n\n      return `![${alt}](${src})\\n\\n${caption}\\n\\n`;\n    },\n  });\n\n  // Use Obsidian format for YouTube embeds and tweets\n  turndownService.addRule(\"embedToMarkdown\", {\n    filter: function (node: Node): boolean {\n      if (node instanceof HTMLIFrameElement) {\n        const src = node.getAttribute(\"src\");\n        return !!src && (\n          !!src.match(/(?:youtube\\.com|youtu\\.be)/) ||\n          !!src.match(/(?:twitter\\.com|x\\.com)/)\n        );\n      }\n      return false;\n    },\n    replacement: function (content: string, node: Node): string {\n      if (node instanceof HTMLIFrameElement) {\n        const src = node.getAttribute(\"src\");\n        if (src) {\n          const youtubeMatch = src.match(\n            /(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com|youtu\\.be)\\/(?:embed\\/|watch\\?v=)?([a-zA-Z0-9_-]+)/,\n          );\n          if (youtubeMatch && youtubeMatch[1]) {\n            return `![](https://www.youtube.com/watch?v=${youtubeMatch[1]})`;\n          }\n          const tweetMatch = src.match(\n            /(?:twitter\\.com|x\\.com)\\/.*?(?:status|statuses)\\/(\\d+)/,\n          );\n          if (tweetMatch && tweetMatch[1]) {\n            return `![](https://x.com/i/status/${tweetMatch[1]})`;\n          }\n        }\n      }\n      return content;\n    },\n  });\n\n  turndownService.addRule(\"highlight\", {\n    filter: \"mark\",\n    replacement: function (content) {\n      return \"==\" + content + \"==\";\n    },\n  });\n\n  turndownService.addRule(\"strikethrough\", {\n    filter: (node: Node) =>\n      node.nodeName === \"DEL\" ||\n      node.nodeName === \"S\" ||\n      node.nodeName === \"STRIKE\",\n    replacement: function (content) {\n      return \"~~\" + content + \"~~\";\n    },\n  });\n\n  // Add a new custom rule for complex link structures\n  turndownService.addRule(\"complexLinkStructure\", {\n    filter: function (node, options) {\n      return (\n        node.nodeName === \"A\" &&\n        node.childNodes.length > 1 &&\n        Array.from(node.childNodes).some((child) =>\n          [\"H1\", \"H2\", \"H3\", \"H4\", \"H5\", \"H6\"].includes(child.nodeName)\n        )\n      );\n    },\n    replacement: function (content, node, options) {\n      if (\n        !(node instanceof HTMLElement) || !(node instanceof HTMLAnchorElement)\n      ) return content;\n      const href = node.getAttribute(\"href\");\n      const title = node.getAttribute(\"title\");\n\n      // Extract the heading\n      const headingNode = node.querySelector(\"h1, h2, h3, h4, h5, h6\");\n      const headingContent = headingNode\n        ? turndownService.turndown(headingNode.innerHTML)\n        : \"\";\n\n      // Remove the heading from the content\n      if (headingNode) {\n        headingNode.remove();\n      }\n\n      // Convert the remaining content\n      const remainingContent = turndownService.turndown(node.innerHTML);\n\n      // Construct the new markdown\n      let markdown = `${headingContent}\\n\\n${remainingContent}\\n\\n`;\n      if (href) {\n        markdown += `[View original](${href})`;\n        if (title) {\n          markdown += ` \"${title}\"`;\n        }\n      }\n\n      return markdown;\n    },\n  });\n\n  turndownService.addRule(\"arXivEnumerate\", {\n    filter: (node) => {\n      return node.nodeName === \"OL\" && node.classList.contains(\"ltx_enumerate\");\n    },\n    replacement: function (content, node) {\n      if (\n        !(node instanceof HTMLElement) || !(node instanceof HTMLOListElement)\n      ) return content;\n\n      const items = Array.from(node.children).map((item, index) => {\n        if (item instanceof HTMLElement) {\n          const itemContent = item.innerHTML.replace(\n            /^

\\d+\\.<\\/span>\\s*/,\n \"\",\n );\n return `${index + 1}. ${turndownService.turndown(itemContent)}`;\n }\n return \"\";\n });\n\n return \"\\n\\n\" + items.join(\"\\n\\n\") + \"\\n\\n\";\n },\n });\n\n turndownService.addRule(\"removeHiddenElements\", {\n filter: function (node) {\n return (\n node.style.display === \"none\"\n );\n },\n replacement: function () {\n return \"\";\n },\n });\n\n turndownService.addRule(\"citations\", {\n filter: (node: Node): boolean => {\n if (node instanceof Element) {\n return (\n node.nodeName === \"SUP\" && node.id.startsWith(\"fnref:\")\n );\n }\n return false;\n },\n replacement: (content, node) => {\n if (node instanceof HTMLElement) {\n if (node.nodeName === \"SUP\" && node.id.startsWith(\"fnref:\")) {\n const id = node.id.replace(\"fnref:\", \"\");\n // Extract only the primary number before any hyphen\n const primaryNumber = id.split(\"-\")[0];\n return `[^${primaryNumber}]`;\n }\n }\n return content;\n },\n });\n\n // Footnotes list\n turndownService.addRule(\"footnotesList\", {\n filter: (node: Node): boolean => {\n if (node instanceof HTMLOListElement) {\n return (\n node.parentElement?.id === \"footnotes\"\n );\n }\n return false;\n },\n replacement: (content, node) => {\n if ((node instanceof HTMLElement) || (node instanceof HTMLOListElement)) {\n const references = Array.from(node.children).map((li) => {\n let id;\n if (li.id.startsWith(\"fn:\")) {\n id = li.id.replace(\"fn:\", \"\");\n } else {\n const match = li.id.split(\"/\").pop()?.match(/cite_note-(.+)/);\n id = match ? match[1] : li.id;\n }\n\n // Remove the leading sup element if its content matches the footnote id\n const supElement = li.querySelector(\"sup\");\n if (supElement && supElement.textContent?.trim() === id) {\n supElement.remove();\n }\n\n const referenceContent = turndownService.turndown(li.innerHTML);\n // Remove the backlink from the footnote content\n const cleanedContent = referenceContent.replace(/\\s*↩︎$/, \"\").trim();\n return `[^${id.toLowerCase()}]: ${cleanedContent}`;\n });\n return \"\\n\\n\" + references.join(\"\\n\\n\") + \"\\n\\n\";\n }\n return content;\n },\n });\n\n // General removal rules for varous website elements\n turndownService.addRule(\"removals\", {\n filter: function (node) {\n if (!(node instanceof HTMLElement)) return false;\n // Remove the Defuddle backlink from the footnote content\n if (node.getAttribute(\"href\")?.includes(\"#fnref\")) return true;\n if (node.classList.contains(\"footnote-backref\")) return true;\n return false;\n },\n replacement: function (content, node) {\n return \"\";\n },\n });\n\n turndownService.addRule(\"handleTextNodesInTables\", {\n filter: function (node: Node): boolean {\n return node.nodeType === Node.TEXT_NODE &&\n node.parentNode !== null &&\n node.parentNode.nodeName === \"TD\";\n },\n replacement: function (content: string): string {\n return content;\n },\n });\n\n turndownService.addRule(\"preformattedCode\", {\n filter: (node) => {\n return node.nodeName === \"PRE\";\n },\n replacement: (content, node) => {\n // if (!(node instanceof HTMLElement) || !(node instanceof HTMLPreElement)) {\n // console.log(node);\n // return content;\n // }\n\n const codeElement = node.querySelector(\"code\");\n if (!codeElement) {\n return content;\n }\n\n const language = codeElement.getAttribute(\"data-lang\") || \"\";\n const code = codeElement.textContent || \"\";\n\n // Clean up the content and escape backticks\n const cleanCode = code\n .trim()\n .replace(/`/g, \"\\\\`\");\n\n return `\\n\\`\\`\\`${language}\\n${cleanCode}\\n\\`\\`\\`\\n`;\n },\n });\n\n turndownService.addRule(\"MathJax\", {\n filter: (node) => {\n const isMjxContainer = node.nodeName.toLowerCase() === \"mjx-container\";\n return isMjxContainer;\n },\n replacement: (content, node) => {\n if (!(node instanceof HTMLElement)) {\n return content;\n }\n\n const assistiveMml = node.querySelector(\"mjx-assistive-mml\");\n if (!assistiveMml) {\n return content;\n }\n\n const mathElement = assistiveMml.querySelector(\"math\");\n if (!mathElement) {\n return content;\n }\n\n let latex;\n try {\n latex = MathMLToLaTeX.convert(mathElement.outerHTML);\n } catch (error) {\n console.error(\"Error converting MathML to LaTeX:\", error);\n return content;\n }\n\n // Check if it's an inline or block math element\n const isBlock = mathElement.getAttribute(\"display\") === \"block\";\n\n if (isBlock) {\n return `\\n$$\\n${latex}\\n$$\\n`;\n } else {\n return `$${latex}$`;\n }\n },\n });\n\n turndownService.addRule(\"math\", {\n filter: (node) => {\n return node.nodeName.toLowerCase() === \"math\" ||\n (node instanceof Element && node.classList &&\n (node.classList.contains(\"mwe-math-element\") ||\n node.classList.contains(\"mwe-math-fallback-image-inline\") ||\n node.classList.contains(\"mwe-math-fallback-image-display\")));\n },\n replacement: (content, node) => {\n if (!(node instanceof Element)) return content;\n\n let latex = extractLatex(node);\n\n // Remove leading and trailing whitespace\n latex = latex.trim();\n\n // Check if the math element is within a table\n const isInTable = node.closest(\"table\") !== null;\n\n // Check if it's an inline or block math element\n if (\n !isInTable && (\n node.getAttribute(\"display\") === \"block\" ||\n node.classList.contains(\"mwe-math-fallback-image-display\") ||\n (node.parentElement &&\n node.parentElement.classList.contains(\"mwe-math-element\") &&\n node.parentElement.previousElementSibling &&\n node.parentElement.previousElementSibling.nodeName.toLowerCase() ===\n \"p\")\n )\n ) {\n return `\\n$$\\n${latex}\\n$$\\n`;\n } else {\n // For inline math, ensure there's a space before and after only if needed\n const prevNode = node.previousSibling;\n const nextNode = node.nextSibling;\n const prevChar = prevNode?.textContent?.slice(-1) || \"\";\n const nextChar = nextNode?.textContent?.[0] || \"\";\n\n const isStartOfLine = !prevNode ||\n (prevNode.nodeType === Node.TEXT_NODE &&\n prevNode.textContent?.trim() === \"\");\n const isEndOfLine = !nextNode ||\n (nextNode.nodeType === Node.TEXT_NODE &&\n nextNode.textContent?.trim() === \"\");\n\n const leftSpace =\n (!isStartOfLine && prevChar && !/[\\s$]/.test(prevChar)) ? \" \" : \"\";\n const rightSpace = (!isEndOfLine && nextChar && !/[\\s$]/.test(nextChar))\n ? \" \"\n : \"\";\n\n return `${leftSpace}$${latex}$${rightSpace}`;\n }\n },\n });\n\n turndownService.addRule(\"katex\", {\n filter: (node) => {\n return node instanceof HTMLElement &&\n (node.classList.contains(\"math\") || node.classList.contains(\"katex\"));\n },\n replacement: (content, node) => {\n if (!(node instanceof HTMLElement)) return content;\n\n // Try to find the original LaTeX content\n // 1. Check data-latex attribute\n let latex = node.getAttribute(\"data-latex\");\n\n // 2. If no data-latex, try to get from .katex-mathml\n if (!latex) {\n const mathml = node.querySelector(\n '.katex-mathml annotation[encoding=\"application/x-tex\"]',\n );\n latex = mathml?.textContent || \"\";\n }\n\n // 3. If still no content, use text content as fallback\n if (!latex) {\n latex = node.textContent?.trim() || \"\";\n }\n\n // Determine if it's an inline formula\n const mathElement = node.querySelector(\".katex-mathml math\");\n const isInline = node.classList.contains(\"math-inline\") ||\n (mathElement && mathElement.getAttribute(\"display\") !== \"block\");\n\n if (isInline) {\n return `$${latex}$`;\n } else {\n return `\\n$$\\n${latex}\\n$$\\n`;\n }\n },\n });\n\n turndownService.addRule(\"callout\", {\n filter: (node) => {\n return (\n node.nodeName.toLowerCase() === \"div\" &&\n node.classList.contains(\"markdown-alert\")\n );\n },\n replacement: (content, node) => {\n const element = node as HTMLElement;\n\n // Get alert type from the class (e.g., markdown-alert-note -> NOTE)\n const alertClasses = Array.from(element.classList);\n const typeClass = alertClasses.find((c) =>\n c.startsWith(\"markdown-alert-\") && c !== \"markdown-alert\"\n );\n const type = typeClass\n ? typeClass.replace(\"markdown-alert-\", \"\").toUpperCase()\n : \"NOTE\";\n\n // Find the title element and content\n const titleElement = element.querySelector(\".markdown-alert-title\");\n const contentElement = element.querySelector(\n \"p:not(.markdown-alert-title)\",\n );\n\n // Extract content, removing the title from it if present\n let alertContent = content;\n if (titleElement && titleElement.textContent) {\n alertContent = contentElement?.textContent ||\n content.replace(titleElement.textContent, \"\");\n }\n\n // Format as Obsidian callout\n return `\\n> [!${type}]\\n> ${\n alertContent.trim().replace(/\\n/g, \"\\n> \")\n }\\n`;\n },\n });\n\n function handleNestedEquations(table: Element): string {\n const mathElements = table.querySelectorAll(\"math[alttext]\");\n if (mathElements.length === 0) return \"\";\n\n return Array.from(mathElements).map((mathElement) => {\n const alttext = mathElement.getAttribute(\"alttext\");\n if (alttext) {\n // Check if it's an inline or block equation\n const isInline = mathElement.closest(\".ltx_eqn_inline\") !== null;\n return isInline ? `$${alttext.trim()}$` : `\\n$$\\n${alttext.trim()}\\n$$`;\n }\n return \"\";\n }).join(\"\\n\\n\");\n }\n\n function cleanupTableHTML(table: HTMLTableElement): string {\n const allowedAttributes = [\n \"src\",\n \"href\",\n \"style\",\n \"align\",\n \"width\",\n \"height\",\n \"rowspan\",\n \"colspan\",\n \"bgcolor\",\n \"scope\",\n \"valign\",\n \"headers\",\n ];\n\n const cleanElement = (element: Element) => {\n Array.from(element.attributes).forEach((attr) => {\n if (!allowedAttributes.includes(attr.name)) {\n element.removeAttribute(attr.name);\n }\n });\n\n element.childNodes.forEach((child) => {\n if (child instanceof Element) {\n cleanElement(child);\n }\n });\n };\n\n // Create a clone of the table to avoid modifying the original DOM\n const tableClone = table.cloneNode(true) as HTMLTableElement;\n cleanElement(tableClone);\n\n return tableClone.outerHTML;\n }\n\n function extractLatex(element: Element): string {\n // Check if the element is a

element and has an alttext attribute\n    if (element.nodeName.toLowerCase() === \"math\") {\n      let latex = element.getAttribute(\"data-latex\");\n      let alttext = element.getAttribute(\"alttext\");\n      if (latex) {\n        return latex.trim();\n      } else if (alttext) {\n        return alttext.trim();\n      }\n      console.log(\"No latex or alttext found for math element:\", element);\n    }\n\n    // If not, look for a nested element with alttext\n    const mathElement = element.querySelector(\"math[alttext]\");\n    if (mathElement) {\n      const alttext = mathElement.getAttribute(\"alttext\");\n      if (alttext) {\n        return alttext.trim();\n      }\n    }\n\n    const annotation = element.querySelector(\n      'annotation[encoding=\"application/x-tex\"]',\n    );\n    if (annotation?.textContent) {\n      return annotation.textContent.trim();\n    }\n\n    const mathNode = element.nodeName.toLowerCase() === \"math\"\n      ? element\n      : element.querySelector(\"math\");\n    if (mathNode) {\n      return MathMLToLaTeX.convert(mathNode.outerHTML);\n    }\n\n    const imgNode = element.querySelector(\"img\");\n    return imgNode?.getAttribute(\"alt\") || \"\";\n  }\n\n  try {\n    let markdown = turndownService.turndown(processedContent);\n    debugLog(\"Markdown\", \"Markdown conversion successful\");\n\n    // Remove the title from the beginning of the content if it exists\n    const titleMatch = markdown.match(/^# .+\\n+/);\n    if (titleMatch) {\n      markdown = markdown.slice(titleMatch[0].length);\n    }\n\n    // Remove any empty links e.g. [](example.com) that remain, along with surrounding newlines\n    // But don't affect image links like ![](image.jpg)\n    markdown = markdown.replace(/\\n*(? 0) {\n      markdown += \"\\n\\n---\\n\\n\";\n      for (const [id, content] of Object.entries(footnotes)) {\n        markdown += `[^${id}]: ${content}\\n\\n`;\n      }\n    }\n\n    // Clear the footnotes object for the next conversion\n    Object.keys(footnotes).forEach((key) => delete footnotes[key]);\n\n    return markdown.trim();\n  } catch (error) {\n    console.error(\"Error converting HTML to Markdown:\", error);\n    console.log(\n      \"Problematic content:\",\n      processedContent.substring(0, 1000) + \"...\",\n    );\n    return `Partial conversion completed with errors. Original HTML:\\n\\n${processedContent}`;\n  }\n}\n\n// let htmlStr =\n//   `

Testing

\"use server-entry\";

export async function Comments() {
  // Load data from a database...
  let comments = await db.getComments();

  return comments.map((comment) => (
    <article key={comment.id}>
      <p>Posted by: {comment.user}</p>
      {renderMarkdown(comment.body)}
    </article>
  ));
}

`;\n// let res = createMarkdownContent(htmlStr, \"http://localhost\");\n\n// console.log(res);\n\nclass ErrorResponse extends Response {\n constructor(error: Error, errorCode: number, resInit: unknown = {}) {\n super(JSON.stringify({ error: error.message, code: errorCode }, null, 2), {\n status: 500,\n ...(resInit ?? {}),\n headers: {\n \"Content-Type\": \"application/json\",\n ...(resInit?.headers ?? {}),\n },\n });\n }\n}\n\nfunction validateInputURL(inputUrl: string | undefined): Response | undefined {\n if (!inputUrl || !URL.canParse(inputUrl)) {\n return new ErrorResponse(\n new Error(\n `No URL provided or could not be parsed, you provided: '${inputUrl}'`,\n ),\n 1,\n );\n }\n}\n\nasync function loadHTML(targetUrl: string): Promise {\n let res = await fetch(targetUrl, {\n method: \"GET\",\n headers: {\n \"User-Agent\": \"Article-Alchemy/1.0 (report issues to matt@hamlin.place)\",\n },\n });\n\n if (!res.ok) {\n return new Error(`Failed to load requested URL, status: ${res.status}`);\n }\n\n if (res.status >= 200 && res.status <= 300) {\n try {\n let content = await res.text();\n\n return content;\n } catch (err) {\n return new Error(\"Failed to parse response as text\", { cause: err });\n }\n }\n\n return new Error(`Received invalid response status code: ${res.status}`);\n}\n\nexport default async function handler(request: Request): Promise {\n switch (request.method) {\n case \"GET\": {\n let reqUrl = new URL(request.url, \"http://localhost\");\n\n let searchParams = new URLSearchParams(reqUrl.search);\n let requestedUrl = searchParams.get(\"url\");\n\n let res = validateInputURL(requestedUrl);\n if (res) {\n return res;\n }\n\n let targetHTMLStringOrError = await loadHTML(requestedUrl);\n\n if (Error.isError(targetHTMLStringOrError)) {\n return new ErrorResponse(targetHTMLStringOrError, 2);\n }\n\n let targetHTMLString = targetHTMLStringOrError as string;\n\n return new Response(\n JSON.stringify(\n {\n content: targetHTMLString,\n },\n null,\n 2,\n ),\n {\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n case \"POST\": {\n let body = await request.json();\n\n let requestedUrl = body.url;\n\n let res = validateInputUrl(requestedUrl);\n if (res) {\n return res;\n }\n\n let targetHTMLStringOrError = await loadHTML(requestedUrl);\n\n if (Error.isError(targetHTMLStringOrError)) {\n return new ErrorResponse(targetHTMLStringOrError, 2);\n }\n\n let targetHTMLString = targetHTMLStringOrError as string;\n\n return new Response(\n JSON.stringify(body, null, 2),\n {\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n }\n}",[],[172],{"_102":181,"_95":182,"_151":183,"_153":-5,"_154":136,"_155":184,"_157":158,"_104":185,"_99":186,"_161":162,"_163":187,"_165":22,"_166":-5,"_167":188,"_98":189},"027f31ea-8d10-11f0-a45f-0224a6c84d84","string-utils.ts","78aa8706-8d0c-11f0-a9be-0224a6c84d84",19,["D",1757376034929],10,"// deno-lint-ignore-file prefer-const\nimport { DOMParser } from \"npm:linkedom\";\n\n// Cases to handle:\n// Full URLs: https://example.com/x.png\n// URLs without protocol: //example.com/x.png\n// Relative URLs:\n// - x.png\n// - /x.png\n// - img/x.png\n// - ../x.png\n\nexport function makeUrlAbsolute(\n element: Element,\n attributeName: string,\n baseUrl: URL,\n) {\n const attributeValue = element.getAttribute(attributeName);\n if (attributeValue) {\n try {\n // Create a new URL object from the base URL\n const resolvedBaseUrl = new URL(baseUrl.href);\n\n // If the base URL points to a file, remove the filename to get the directory\n if (!resolvedBaseUrl.pathname.endsWith(\"/\")) {\n resolvedBaseUrl.pathname = resolvedBaseUrl.pathname.substring(\n 0,\n resolvedBaseUrl.pathname.lastIndexOf(\"/\") + 1,\n );\n }\n\n const url = new URL(attributeValue, resolvedBaseUrl);\n\n if (![\"http:\", \"https:\"].includes(url.protocol)) {\n // Handle non-standard protocols (chrome-extension://, moz-extension://, brave://, etc.)\n const parts = attributeValue.split(\"/\");\n const firstSegment = parts[2]; // The segment after the protocol\n\n if (firstSegment && firstSegment.includes(\".\")) {\n // If it looks like a domain, replace the non-standard protocol with the current page's protocol\n const newUrl = `${baseUrl.protocol}//` +\n attributeValue.split(\"://\")[1];\n element.setAttribute(attributeName, newUrl);\n } else {\n // If it doesn't look like a domain it's probably the extension URL, remove the non-standard protocol part and use baseUrl\n const path = parts.slice(3).join(\"/\");\n const newUrl =\n new URL(path, resolvedBaseUrl.origin + resolvedBaseUrl.pathname)\n .href;\n element.setAttribute(attributeName, newUrl);\n }\n } else {\n // Handle other cases (relative URLs, protocol-relative URLs)\n const newUrl = url.href;\n element.setAttribute(attributeName, newUrl);\n }\n } catch (error) {\n console.warn(`Failed to process URL: ${attributeValue}`, error);\n element.setAttribute(attributeName, attributeValue);\n }\n }\n}\n\n// @todo rewrite to use linkedom DOMParser\nexport function processUrls(htmlContent: string, baseUrl: URL): string {\n let parser = new DOMParser();\n let doc = parser.parseFromString(htmlContent, \"text/html\");\n\n // const tempDiv = document.createElement(\"div\");\n // tempDiv.innerHTML = htmlContent;\n\n // Handle relative URLs for both images and links\n doc.querySelectorAll(\"img\").forEach((img) =>\n makeUrlAbsolute(img, \"srcset\", baseUrl)\n );\n doc.querySelectorAll(\"img\").forEach((img) =>\n makeUrlAbsolute(img, \"src\", baseUrl)\n );\n doc.querySelectorAll(\"a\").forEach((link) =>\n makeUrlAbsolute(link, \"href\", baseUrl)\n );\n\n return doc.toString();\n}",[],[182],"files",[148,170,180],"counts",{"_194":32,"_195":196,"_197":32,"_198":199},"pulls","branches",1,"forks","envVars",{"_200":32,"_201":32},"count","emptyCount","contents",[204,207,210],{"_102":149,"_95":150,"_151":152,"_153":-5,"_154":136,"_155":156,"_157":158,"_104":205,"_99":160,"_161":162,"_163":164,"_165":22,"_166":-5,"_98":206},["D",1758282055828],[150],{"_102":171,"_95":172,"_151":173,"_153":-5,"_154":136,"_155":135,"_157":158,"_104":208,"_99":175,"_161":176,"_163":177,"_165":22,"_166":-5,"_98":209},["D",1758295380994],[172],{"_102":181,"_95":182,"_151":183,"_153":-5,"_154":136,"_155":184,"_157":158,"_104":211,"_99":186,"_161":162,"_163":187,"_165":22,"_166":-5,"_98":212},["D",1757376034929],[182],"file","upstreamProject","upstreamBranch","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.pulls",{"_93":94,"_95":96,"_222":223,"_100":226,"_194":231,"_232":-5,"_233":22,"_234":-5},"filters",{"_224":225},"state","open",{"_102":103,"_95":96,"_104":227,"_106":-5,"_107":22,"_108":109,"_110":-5,"_111":-5,"_14":112,"_113":-5,"_114":22,"_115":-5,"_116":32,"_16":117,"_118":119,"_120":119,"_121":122,"_123":122,"_124":122,"_125":119,"_126":119,"_23":228,"_130":229,"_133":230,"_145":22},["D",1757374404832],{"_93":94,"_128":129},{"_93":94,"_132":129},{"_99":135,"_95":43,"_102":136,"_137":103,"_138":-5,"_139":-5,"_140":-5,"_141":-5,"_142":-5,"_143":96,"_144":94},[],"latestBranch","isForkOwner","ownProjectForks","actionData","errors"]

matthamlin

html-to-markdown-experiments

Pull requests