[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_164":165,"_166":167},"routes/_app.x.$handle.$name",{"_95":94,"_32":45,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":49,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",225,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","0b9ea5ea-2793-11f0-9d52-569c3dd06744","name","fileType","http","content","/** @jsx jsx */\nimport { jsx } from \"https://deno.land/x/hono@v3.11.7/middleware.ts\";\nimport { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { ai } from \"https://esm.town/v/yawnxyz/ai\";\nimport { getUrlMetadata } from \"https://esm.town/v/yawnxyz/urlMetadata\";\nimport { getHtmlMetadata } from \"https://esm.town/v/yawnxyz/htmlMetadata\";\nimport { getCitation } from \"https://esm.town/v/yawnxyz/citation\";\nimport { blobby } from \"https://esm.town/v/@yawnxyz/blobby\";\nimport stringHash from 'npm:string-hash';\n\nconst app = new Hono();\n\n// https://www.crossref.org/blog/dois-and-matching-regular-expressions/\nconst DOI_REGEX = /\\b(10\\.\\d{4,9}\\/[-._;()\\/:\\w]+)\\b/i;\n\nexport const getJinaContent = async (url, opts = {}) => {\n const baseUrl = 'https://r.jina.ai/';\n if (!url.includes('r.jina.ai')) {\n url = baseUrl + url;\n }\n const fullUrl = new URL(url);\n\n const headers = {\n ...(opts.withImagesSummary && { 'X-With-Images-Summary': 'true' }),\n ...(opts.withGeneratedAlt && { 'X-With-Generated-Alt': 'true' }),\n ...(opts.withLinksSummary && { 'X-With-Links-Summary': 'true' }),\n ...(opts.noCache && { 'X-No-Cache': 'true' }),\n ...(opts.accept && { 'Accept': opts.accept }),\n ...(opts.targetSelector && { 'X-Target-Selector': opts.targetSelector }),\n ...(opts.timeout && { 'X-Timeout': opts.timeout.toString() }),\n ...(opts.waitForSelector && { 'X-Wait-For-Selector': opts.waitForSelector }),\n ...(opts.returnFormat && { 'X-Return-Format': opts.returnFormat }),\n };\n\n console.log('[getJinaContent] Fetching:', fullUrl.toString(), headers);\n\n try {\n const response = await fetch(fullUrl.toString(), {\n method: 'GET',\n headers: headers,\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n if (headers['Accept'] === 'text/event-stream') {\n return response.body;\n }\n\n const text = await response.text();\n try {\n return JSON.parse(text);\n } catch {\n if (headers['X-Return-Format'] === 'screenshot') {\n return JSON.parse(text);\n } else {\n return text;\n }\n }\n } catch (error) {\n console.error('Error fetching from Jina:', error);\n throw error;\n }\n}\n\nexport const getFirecrawlContent = async (url, opts = {}) => {\n const apiKey = Deno.env.get(\"FIRECRAWL_API_KEY\");\n if (!apiKey) {\n throw new Error(\"API key not found. Please set FIRECRAWL_API_KEY in your environment.\");\n }\n\n const body = {\n url: url,\n pageOptions: {\n onlyMainContent: opts.onlyMainContent || true,\n includeHtml: opts.includeHtml || false,\n includeRawHtml: opts.includeRawHtml || false,\n screenshot: opts.screenshot || false,\n waitFor: opts.waitFor || 0,\n removeTags: opts.removeTags || [],\n onlyIncludeTags: opts.onlyIncludeTags || [],\n replaceAllPathsWithAbsolutePaths: opts.replaceAllPathsWithAbsolutePaths || true,\n parsePDF: opts.parsePDF || false\n },\n extractorOptions: {\n // mode: opts.mode || \"llm-extraction\",\n mode: opts.mode,\n extractionPrompt: opts.extractionPrompt || \"\",\n extractionSchema: opts.extractionSchema || {}\n },\n timeout: opts.timeout || 30000\n };\n\n console.log('[getFirecrawlContent] Fetching:', url, body);\n\n try {\n const response = await fetch(\"https://api.firecrawl.dev/v0/scrape\", {\n method: 'POST',\n headers: {\n \"Authorization\": `Bearer ${apiKey}`,\n \"Content-Type\": \"application/json\"\n },\n body: JSON.stringify(body)\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const result = await response.json();\n return result;\n } catch (error) {\n console.error('Error fetching from Firecrawl:', error);\n throw error;\n }\n};\n\nexport const getContentFromUrl_v1 = async (url: string) => {\n try {\n let ogUrl = url;\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n url = \"https://r.jina.ai/\" + url;\n\n let [response, metadata] = await Promise.all([\n fetch(url),\n getUrlMetadata(ogUrl)\n ]);\n\n const contentTextData = await response.text();\n\n if (!metadata) {\n const html = await getJinaContent(url, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n // console.log('---> metadata?? ', metadata, ' url:' + url)\n let doi\n\n if (metadata) {\n const doiRegex = /10.\\d{4,9}\\/[-._;()\\/:A-Z0-9]+/gi;\n\n\n const doiMatch = JSON.stringify(metadata).match(doiRegex);\n doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n }\n\n console.log('DOI found?:', doi);\n let citation\n\n if (doi) {\n citation = await getCitation(doi);\n } else {\n }\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n let output = \"\"\n if(doi) {\n output = output + \"\\n \" + doi + \"\\n\\n\\n\"\n }\n if(citation) {\n output = output + \"\\n \" + citation + \"\\n\\n\\n\"\n }\n if(contentTextData) {\n output = output + \"\\n \" + contentTextData + \"\\n\\n\\n\"\n }\n if(metadata) {\n output = output + \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\"\n }\n return output\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content.';\n }\n};\n\nexport const extractDOI = (content) => {\n if(!content) return null;\n if(!content.match) return null;\n \n const doiRegex = DOI_REGEX;\n const doiMatch = content.match(doiRegex);\n let doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n return doi;\n};\n\n\n\n\nexport const getContentFromUrl = async ({ url, optionStr = 'content', crawlMode = 'jina', useCache = true }) => {\n try {\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n let ogUrl = url;\n let urlObject = new URL(url);\n if (urlObject.href.includes('&opts=')) { // Check if the URL contains '&opts='\n // Remove everything from '&opts=' to the end of the string\n // *** this might be brittle!!\n urlObject.href = urlObject.href.split('&opts=')[0];\n }\n\n url = urlObject.toString();\n let str = `${url}_${optionStr}_${crawlMode}`;\n const cacheId = `cache:${stringHash(str)}`;\n\n console.log('[gCFU] useCache:', useCache, 'cacheId:', cacheId)\n\n if (useCache) {\n const cachedContent = await blobby.get(cacheId);\n if (cachedContent) {\n console.log('[gCFU] cache hit:', cacheId);\n return cachedContent;\n }\n }\n\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n const serviceUrl = crawlMode == 'fire' ? url : \"https://r.jina.ai/\" + url;\n\n console.log('getting content from url', serviceUrl);\n\n const optsArray = optionStr.split(',');\n const requestOptions = {\n returnFormat: optsArray.includes('html') ? 'html' : optsArray.includes('json') ? 'json' : undefined,\n withImagesSummary: optsArray.includes('images'),\n withLinksSummary: optsArray.includes('links'),\n withGeneratedAlt: optsArray.includes('caption')\n };\n\n const fetchMetadata = optsArray.includes('metadata') || optsArray.includes('doi') || optsArray.includes('citation');\n let metadata;\n\n if (fetchMetadata) {\n metadata = await getUrlMetadata(ogUrl);\n }\n\n\n const content = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, requestOptions)\n : await getJinaContent(serviceUrl, requestOptions);\n\n if (fetchMetadata && !metadata) {\n const html = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, { returnFormat: 'html' })\n : await getJinaContent(serviceUrl, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n if(!metadata || Object.keys(metadata).length === 0) {\n metadata = content; // for pdfs, it'll just have to be the content\n }\n\n let doi;\n if (optsArray.includes('doi') || optsArray.includes('citation')) {\n if (metadata) {\n doi = extractDOI(JSON.stringify(metadata));\n }\n if (!doi) {\n doi = extractDOI(content);\n }\n if (!doi) {\n doi = extractDOI(metadata);\n }\n\n console.log('DOI found?:', doi);\n }\n\n let citation;\n if (optsArray.includes('citation') && doi) {\n citation = await getCitation(doi);\n }\n\n let output = \"\";\n output += \"\" + serviceUrl + \"\" + \" \";\n output += \"\" + cacheId + \"\";\n if (optsArray.includes('doi') && doi) {\n output += \"\\n \" + doi + \"\\n\\n\\n\";\n }\n if (optsArray.includes('citation') && citation) {\n output += \"\\n \" + citation + \"\\n\\n\\n\";\n }\n if (optsArray.includes('content') && content) {\n output += \"\\n \" + content + \"\\n\\n\\n\";\n }\n if (optsArray.includes('metadata') && metadata) {\n output += \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\";\n }\n\n if (useCache) {\n await blobby.set(cacheId, output);\n console.log('Cache set:', cacheId);\n }\n\n if (output.length === 0) {\n output = '(This area intentionally left blank)';\n }\n\n return output;\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content.';\n }\n};\n\n\napp.get(\"/*\", async (c) => {\n let url = c.req.query(\"url\");\n let opts = c.req.query(\"opts\") || 'content';\n let crawlMode = c.req.query(\"crawlMode\") === 'true';\n let noCache = c.req.query(\"noCache\") === 'true';\n let useCache = !noCache;\n\n if (!url) {\n const rawUrl = c.req.raw.url;\n const pathStartIndex = rawUrl.indexOf('/', rawUrl.indexOf('://') + 3);\n url = rawUrl.slice(pathStartIndex + 1); // Get the full path after the initial '/'\n }\n\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n return c.text(content);\n});\n\n\napp.post(\"/\", async (c) => {\n const body = await c.req.json();\n const { url, opts = 'content', crawlMode = 'jina', useCache, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body;\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n\n if (!getSummary && !getTags) {\n return c.text(content);\n }\n\n const [summary, tags, embeddings] = await Promise.all([\n getSummary ? getSummaryFn(getSummaryPrompt, content) : null,\n getTags ? getTagsFn(getTagsPrompt, content) : null,\n getEmbeddings ? getEmbeddingsFn(content) : null\n ]);\n\n return c.json({ summary: summary?.text, tags: tags?.text, content, embeddings });\n});\n\n\n\n\n\n\n\n\n\n\nexport const getBlurbFromUrl = async (url: string, { content,\n getSummary = true, getSummaryPrompt, getTags = true, getTagsPrompt, getEmbeddings,\n model, provider\n} = {}) => {\n content = content || await getContentFromUrl_v1(url);\n\n if (!getSummary || !getTags) {\n return content;\n }\n\n let summary, tags, embeddings\n if (getSummary) {\n summary = await getSummaryFn(getSummaryPrompt, content, { model, provider });\n }\n if (getTags) {\n tags = await getTagsFn(getTagsPrompt, content, { model, provider });\n }\n if (getEmbeddings) {\n embeddings = await getEmbeddingsFn(content + \" \" + summary + \" \" + tags);\n }\n\n return {\n content,\n summary: summary.text,\n tags: tags.text,\n embeddings\n }\n};\n\nexport const getSummaryFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Summarize this article in 30 words, focusing on the primary author. Do not respond with JSON or XML or stage instructions. Only reply in text. Text:\"\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getTagsFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Return a comma separated list of 3-4 tags that broadly describes the text. Do not repeat the given text. Just return text and commas, no other symbols. Text to categorize:\"\n console.log(\"prompt\", prompt)\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getEmbeddingsFn = async (content) => {\n let result = await ai({\n provider: \"openai\",\n embed: true,\n value: content\n })\n return result\n}\n\n\n\n\n\nexport default app.fetch;\n","/** @jsx jsx */\nimport { jsx } from \"https://deno.land/x/hono@v3.11.7/middleware.ts\";\nimport { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { ai } from \"https://esm.town/v/yawnxyz/ai\";\nimport { getUrlMetadata } from \"https://esm.town/v/yawnxyz/urlMetadata\";\nimport { getHtmlMetadata } from \"https://esm.town/v/yawnxyz/htmlMetadata\";\nimport { getCitation } from \"https://esm.town/v/yawnxyz/citation\";\nimport { blobby } from \"https://esm.town/v/@yawnxyz/blobby\";\nimport stringHash from 'npm:string-hash';\n\nconst app = new Hono();\n\n// https://www.crossref.org/blog/dois-and-matching-regular-expressions/\nconst DOI_REGEX = /\\b(10\\.\\d{4,9}\\/[-._;()\\/:\\w]+)\\b/i;\n\nexport const getJinaContent = async (url, opts = {}) => {\n const baseUrl = 'https://r.jina.ai/';\n if (!url.includes('r.jina.ai')) {\n url = baseUrl + url;\n }\n const fullUrl = new URL(url);\n\n const headers = {\n ...(opts.withImagesSummary && { 'X-With-Images-Summary': 'true' }),\n ...(opts.withGeneratedAlt && { 'X-With-Generated-Alt': 'true' }),\n ...(opts.withLinksSummary && { 'X-With-Links-Summary': 'true' }),\n ...(opts.noCache && { 'X-No-Cache': 'true' }),\n ...(opts.accept && { 'Accept': opts.accept }),\n ...(opts.targetSelector && { 'X-Target-Selector': opts.targetSelector }),\n ...(opts.timeout && { 'X-Timeout': opts.timeout.toString() }),\n ...(opts.waitForSelector && { 'X-Wait-For-Selector': opts.waitForSelector }),\n ...(opts.returnFormat && { 'X-Return-Format': opts.returnFormat }),\n };\n\n console.log('[getJinaContent] Fetching:', fullUrl.toString(), headers);\n\n try {\n const response = await fetch(fullUrl.toString(), {\n method: 'GET',\n headers: headers,\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n if (headers['Accept'] === 'text/event-stream') {\n return response.body;\n }\n\n const text = await response.text();\n try {\n return JSON.parse(text);\n } catch {\n if (headers['X-Return-Format'] === 'screenshot') {\n return JSON.parse(text);\n } else {\n return text;\n }\n }\n } catch (error) {\n console.error('Error fetching from Jina:', error);\n throw error;\n }\n}\n\nexport const getFirecrawlContent = async (url, opts = {}) => {\n const apiKey = Deno.env.get(\"FIRECRAWL_API_KEY\");\n if (!apiKey) {\n throw new Error(\"API key not found. Please set FIRECRAWL_API_KEY in your environment.\");\n }\n\n const body = {\n url: url,\n pageOptions: {\n onlyMainContent: opts.onlyMainContent || true,\n includeHtml: opts.includeHtml || false,\n includeRawHtml: opts.includeRawHtml || false,\n screenshot: opts.screenshot || false,\n waitFor: opts.waitFor || 0,\n removeTags: opts.removeTags || [],\n onlyIncludeTags: opts.onlyIncludeTags || [],\n replaceAllPathsWithAbsolutePaths: opts.replaceAllPathsWithAbsolutePaths || true,\n parsePDF: opts.parsePDF || false\n },\n extractorOptions: {\n // mode: opts.mode || \"llm-extraction\",\n mode: opts.mode,\n extractionPrompt: opts.extractionPrompt || \"\",\n extractionSchema: opts.extractionSchema || {}\n },\n timeout: opts.timeout || 30000\n };\n\n console.log('[getFirecrawlContent] Fetching:', url, body);\n\n try {\n const response = await fetch(\"https://api.firecrawl.dev/v0/scrape\", {\n method: 'POST',\n headers: {\n \"Authorization\": `Bearer ${apiKey}`,\n \"Content-Type\": \"application/json\"\n },\n body: JSON.stringify(body)\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const result = await response.json();\n return result;\n } catch (error) {\n console.error('Error fetching from Firecrawl:', error);\n throw error;\n }\n};\n\nexport const getContentFromUrl_v1 = async (url: string) => {\n try {\n let ogUrl = url;\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n url = \"https://r.jina.ai/\" + url;\n\n let [response, metadata] = await Promise.all([\n fetch(url),\n getUrlMetadata(ogUrl)\n ]);\n\n const contentTextData = await response.text();\n\n if (!metadata) {\n const html = await getJinaContent(url, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n // console.log('---> metadata?? ', metadata, ' url:' + url)\n let doi\n\n if (metadata) {\n const doiRegex = /10.\\d{4,9}\\/[-._;()\\/:A-Z0-9]+/gi;\n\n\n const doiMatch = JSON.stringify(metadata).match(doiRegex);\n doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n }\n\n console.log('DOI found?:', doi);\n let citation\n\n if (doi) {\n citation = await getCitation(doi);\n } else {\n }\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n let output = \"\"\n if(doi) {\n output = output + \"\\n \" + doi + \"\\n\\n\\n\"\n }\n if(citation) {\n output = output + \"\\n \" + citation + \"\\n\\n\\n\"\n }\n if(contentTextData) {\n output = output + \"\\n \" + contentTextData + \"\\n\\n\\n\"\n }\n if(metadata) {\n output = output + \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\"\n }\n return output\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content.';\n }\n};\n\nexport const extractDOI = (content) => {\n if(!content) return null;\n if(!content.match) return null;\n \n const doiRegex = DOI_REGEX;\n const doiMatch = content.match(doiRegex);\n let doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n return doi;\n};\n\n\n\n\nexport const getContentFromUrl = async ({ url, optionStr = 'content', crawlMode = 'jina', useCache = true }) => {\n try {\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n let ogUrl = url;\n let urlObject = new URL(url);\n if (urlObject.href.includes('&opts=')) { // Check if the URL contains '&opts='\n // Remove everything from '&opts=' to the end of the string\n // *** this might be brittle!!\n urlObject.href = urlObject.href.split('&opts=')[0];\n }\n\n url = urlObject.toString();\n let str = `${url}_${optionStr}_${crawlMode}`;\n const cacheId = `cache:${stringHash(str)}`;\n\n console.log('[gCFU] useCache:', useCache, 'cacheId:', cacheId)\n\n if (useCache) {\n const cachedContent = await blobby.get(cacheId);\n if (cachedContent) {\n console.log('[gCFU] cache hit:', cacheId);\n return cachedContent;\n }\n }\n\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n const serviceUrl = crawlMode == 'fire' ? url : \"https://r.jina.ai/\" + url;\n\n console.log('getting content from url', serviceUrl);\n\n const optsArray = optionStr.split(',');\n const requestOptions = {\n returnFormat: optsArray.includes('html') ? 'html' : optsArray.includes('json') ? 'json' : undefined,\n withImagesSummary: optsArray.includes('images'),\n withLinksSummary: optsArray.includes('links'),\n withGeneratedAlt: optsArray.includes('caption')\n };\n\n const fetchMetadata = optsArray.includes('metadata') || optsArray.includes('doi') || optsArray.includes('citation');\n let metadata;\n\n if (fetchMetadata) {\n metadata = await getUrlMetadata(ogUrl);\n }\n\n\n const content = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, requestOptions)\n : await getJinaContent(serviceUrl, requestOptions);\n\n if (fetchMetadata && !metadata) {\n const html = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, { returnFormat: 'html' })\n : await getJinaContent(serviceUrl, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n if(!metadata || Object.keys(metadata).length === 0) {\n metadata = content; // for pdfs, it'll just have to be the content\n }\n\n let doi;\n if (optsArray.includes('doi') || optsArray.includes('citation')) {\n if (metadata) {\n doi = extractDOI(JSON.stringify(metadata));\n }\n if (!doi) {\n doi = extractDOI(content);\n }\n if (!doi) {\n doi = extractDOI(metadata);\n }\n\n console.log('DOI found?:', doi);\n }\n\n let citation;\n if (optsArray.includes('citation') && doi) {\n citation = await getCitation(doi);\n }\n\n let output = \"\";\n output += \"\" + serviceUrl + \"\" + \" \";\n output += \"\" + cacheId + \"\";\n if (optsArray.includes('doi') && doi) {\n output += \"\\n \" + doi + \"\\n\\n\\n\";\n }\n if (optsArray.includes('citation') && citation) {\n output += \"\\n \" + citation + \"\\n\\n\\n\";\n }\n if (optsArray.includes('content') && content) {\n output += \"\\n \" + content + \"\\n\\n\\n\";\n }\n if (optsArray.includes('metadata') && metadata) {\n output += \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\";\n }\n\n if (useCache) {\n await blobby.set(cacheId, output);\n console.log('Cache set:', cacheId);\n }\n\n if (output.length === 0) {\n output = '(This area intentionally left blank)';\n }\n\n return output;\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content.';\n }\n};\n\n\napp.get(\"/*\", async (c) => {\n let url = c.req.query(\"url\");\n let opts = c.req.query(\"opts\") || 'content';\n let crawlMode = c.req.query(\"crawlMode\") === 'jina';\n let noCache = c.req.query(\"noCache\") === 'true';\n let useCache = !noCache;\n\n if (!url) {\n const rawUrl = c.req.raw.url;\n const pathStartIndex = rawUrl.indexOf('/', rawUrl.indexOf('://') + 3);\n url = rawUrl.slice(pathStartIndex + 1); // Get the full path after the initial '/'\n }\n\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n return c.text(content);\n});\n\n\napp.post(\"/\", async (c) => {\n const body = await c.req.json();\n const { url, opts = 'content', crawlMode = 'jina', useCache, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body;\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n\n if (!getSummary && !getTags) {\n return c.text(content);\n }\n\n const [summary, tags, embeddings] = await Promise.all([\n getSummary ? getSummaryFn(getSummaryPrompt, content) : null,\n getTags ? getTagsFn(getTagsPrompt, content) : null,\n getEmbeddings ? getEmbeddingsFn(content) : null\n ]);\n\n return c.json({ summary: summary?.text, tags: tags?.text, content, embeddings });\n});\n\n\n\n\n\n\n\n\n\n\nexport const getBlurbFromUrl = async (url: string, { content,\n getSummary = true, getSummaryPrompt, getTags = true, getTagsPrompt, getEmbeddings,\n model, provider\n} = {}) => {\n content = content || await getContentFromUrl_v1(url);\n\n if (!getSummary || !getTags) {\n return content;\n }\n\n let summary, tags, embeddings\n if (getSummary) {\n summary = await getSummaryFn(getSummaryPrompt, content, { model, provider });\n }\n if (getTags) {\n tags = await getTagsFn(getTagsPrompt, content, { model, provider });\n }\n if (getEmbeddings) {\n embeddings = await getEmbeddingsFn(content + \" \" + summary + \" \" + tags);\n }\n\n return {\n content,\n summary: summary.text,\n tags: tags.text,\n embeddings\n }\n};\n\nexport const getSummaryFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Summarize this article in 30 words, focusing on the primary author. Do not respond with JSON or XML or stage instructions. Only reply in text. Text:\"\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getTagsFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Return a comma separated list of 3-4 tags that broadly describes the text. Do not repeat the given text. Just return text and commas, no other symbols. Text to categorize:\"\n console.log(\"prompt\", prompt)\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getEmbeddingsFn = async (content) => {\n let result = await ai({\n provider: \"openai\",\n embed: true,\n value: content\n })\n return result\n}\n\n\n\n\n\nexport default app.fetch;\n","createdAt",["D",1721346099702],"branchVersion","branchId","0b9c2cd4-2793-11f0-9d52-569c3dd06744","id","0b9beaa8-2793-11f0-9d52-569c3dd06744","getContentFromUrl",["D",1746217295690],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",1,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"isOwned",256,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":160,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":161,"_79":162,"_81":163,"_83":25},"tree",[151,152],"files",[151,152],"counts",{"_141":139,"_142":139,"_143":139,"_144":145,"_146":147},"contents",[126,127],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_31":125},"content.coversheet.org",{"_43":135,"_32":136,"_30":137,"_129":-5,"_41":42,"_40":61,"_130":49,"_38":138,"_13":139,"_33":20,"_35":-5,"_24":25,"_133":-5,"_27":140},{"_43":128,"_32":28,"_30":31,"_129":-5,"_41":42,"_40":14,"_130":25,"_38":131,"_13":132,"_33":34,"_35":-5,"_24":25,"_133":-5,"_27":134},"108821b2-2793-11f0-9d52-569c3dd06744","parentId","latest",["D",1721346099702],224,"label",[28],"0b9eaa5e-2793-11f0-9d52-569c3dd06744","README.md","0b9eaacc-2793-11f0-9d52-569c3dd06744",["D",1718163352826],0,[136],"pulls","branches","forks","envVars",{"_148":139,"_150":139},"connections",{"_148":139,"_149":139},"count","unfilledCount","emptyCount",{"_43":135,"_32":136,"_30":137,"_129":-5,"_41":42,"_40":61,"_130":49,"_38":157,"_13":139,"_33":20,"_35":-5,"_24":25,"_133":-5,"_154":158,"_27":159},{"_43":128,"_32":28,"_30":31,"_129":-5,"_41":42,"_40":14,"_130":25,"_38":153,"_13":132,"_33":34,"_35":-5,"_24":25,"_133":-5,"_154":155,"_27":156},["D",1721346099702],"children",[],[28],["D",1718163352826],[],[136],["D",1746217295690],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"ctx",{"_77":-5,"_197":198,"_122":199,"_200":201,"_202":203,"_204":25,"_205":25,"_206":25,"_207":49,"_208":-5,"_209":-7,"_210":-7,"_211":139,"_212":25,"_213":25,"_214":-5},"ENV",{"_168":169,"_170":171,"_172":85,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192,"_193":194,"_195":196},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","db46cea2d909ccbd011cfec0e4ddfe291f419728","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","aiChatModels",[215,216,217,218,219],[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","navCollapsed","currentOrgId",{"_220":244,"_133":245,"_223":246,"_53":247,"_226":49,"_227":49,"_228":229,"_230":49},{"_220":239,"_133":240,"_223":241,"_53":242,"_226":49,"_227":25,"_228":243,"_230":25},{"_220":235,"_133":236,"_223":237,"_53":238,"_226":49,"_227":49,"_228":229,"_230":25},{"_220":231,"_133":232,"_223":233,"_53":234,"_226":49,"_227":49,"_228":229,"_230":25},{"_220":221,"_133":222,"_223":224,"_53":225,"_226":49,"_227":49,"_228":229,"_230":25},"value","fable","Claude Fable 5.1","shortLabel","Fable","Most capable; 2× Opus pricing","imageInput","thinking","contextWindowSize",1000000,"isDefault","opus55","Claude Opus 5.5","Opus","Highly capable for complex work","sonnet55","Claude Sonnet 5.5","Sonnet","Balanced speed and capability","haiku","Claude Haiku 4.5","Haiku","Fastest; best for simple edits",200000,"deepseekflash","DeepSeek V4.1 Flash","DeepSeek Flash","Smart and very cheap"]

Diff · v225