[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_155":156,"_157":158},"routes/_app.x.$handle.$name",{"_93":92,"_31":44,"_99":83,"_26":100,"_13":14,"_11":101,"_102":103,"_104":105,"_106":107,"_108":109,"_20":-5,"_110":-5,"_111":-5,"_112":25,"_113":114,"_115":25},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_31":44,"_37":45,"_46":-5,"_47":48,"_49":50,"_51":-5,"_52":-5,"_18":53,"_54":30,"_55":48,"_56":-5,"_57":-5,"_58":59,"_60":61,"_62":-5,"_63":-5,"_64":-5,"_65":-5,"_66":67,"_68":67,"_69":70,"_71":70,"_72":70,"_73":67,"_74":67,"_75":76,"_77":78,"_79":80,"_81":25},"version",240,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":27,"_28":27},"file",{"_29":30,"_31":27,"_32":33,"_34":36,"_37":38,"_39":14,"_40":41},"previous",{"_29":30,"_31":27,"_32":33,"_34":35},"deleted",false,"path","main.tsx","latestPath","fileId","0b9ea5ea-2793-11f0-9d52-569c3dd06744","name","fileType","http","content","/** @jsx jsx */\nimport { jsx } from \"https://deno.land/x/hono@v3.11.7/middleware.ts\";\nimport { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { ai } from \"https://esm.town/v/yawnxyz/ai\";\nimport { getUrlMetadata } from \"https://esm.town/v/yawnxyz/urlMetadata\";\nimport { getHtmlMetadata } from \"https://esm.town/v/yawnxyz/htmlMetadata\";\nimport { getCitation } from \"https://esm.town/v/yawnxyz/citation\";\nimport { blobby } from \"https://esm.town/v/@yawnxyz/blobby\";\nimport stringHash from 'npm:string-hash';\n\nconst app = new Hono();\n\n// https://www.crossref.org/blog/dois-and-matching-regular-expressions/\nconst DOI_REGEX = /\\b(10\\.\\d{4,9}\\/[-._;()\\/:\\w]+)\\b/i;\n\nexport const getJinaContent = async (url, opts = {}) => {\n const baseUrl = 'https://r.jina.ai/';\n if (!url.includes('r.jina.ai')) {\n url = baseUrl + url;\n }\n const fullUrl = new URL(url);\n\n const headers = {\n ...(opts.withImagesSummary && { 'X-With-Images-Summary': 'true' }),\n ...(opts.withGeneratedAlt && { 'X-With-Generated-Alt': 'true' }),\n ...(opts.withLinksSummary && { 'X-With-Links-Summary': 'true' }),\n ...(opts.noCache && { 'X-No-Cache': 'true' }),\n ...(opts.accept && { 'Accept': opts.accept }),\n ...(opts.targetSelector && { 'X-Target-Selector': opts.targetSelector }),\n ...(opts.timeout && { 'X-Timeout': opts.timeout.toString() }),\n ...(opts.waitForSelector && { 'X-Wait-For-Selector': opts.waitForSelector }),\n ...(opts.returnFormat && { 'X-Return-Format': opts.returnFormat }),\n };\n\n console.log('[getJinaContent] Fetching:', fullUrl.toString(), headers);\n\n try {\n const response = await fetch(fullUrl.toString(), {\n method: 'GET',\n headers: headers,\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n if (headers['Accept'] === 'text/event-stream') {\n return response.body;\n }\n\n const text = await response.text();\n try {\n return JSON.parse(text);\n } catch {\n if (headers['X-Return-Format'] === 'screenshot') {\n return JSON.parse(text);\n } else {\n return text;\n }\n }\n } catch (error) {\n console.error('Error fetching from Jina:', error);\n throw error;\n }\n}\n\nexport const getFirecrawlContent = async (url, opts = {}) => {\n const apiKey = Deno.env.get(\"FIRECRAWL_API_KEY\");\n if (!apiKey) {\n throw new Error(\"API key not found. Please set FIRECRAWL_API_KEY in your environment.\");\n }\n\n const body = {\n url: url,\n pageOptions: {\n onlyMainContent: opts.onlyMainContent || true,\n includeHtml: opts.includeHtml || false,\n includeRawHtml: opts.includeRawHtml || false,\n screenshot: opts.screenshot || false,\n waitFor: opts.waitFor || 0,\n removeTags: opts.removeTags || [],\n onlyIncludeTags: opts.onlyIncludeTags || [],\n replaceAllPathsWithAbsolutePaths: opts.replaceAllPathsWithAbsolutePaths || true,\n parsePDF: opts.parsePDF || false\n },\n extractorOptions: {\n // mode: opts.mode || \"llm-extraction\",\n mode: opts.mode,\n extractionPrompt: opts.extractionPrompt || \"\",\n extractionSchema: opts.extractionSchema || {}\n },\n timeout: opts.timeout || 30000\n };\n\n console.log('[getFirecrawlContent] Fetching:', url, body);\n\n try {\n const response = await fetch(\"https://api.firecrawl.dev/v0/scrape\", {\n method: 'POST',\n headers: {\n \"Authorization\": `Bearer ${apiKey}`,\n \"Content-Type\": \"application/json\"\n },\n body: JSON.stringify(body)\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const result = await response.json();\n let resultText = JSON.stringify(result);\n console.log('[getFirecrawlContent] Result:', resultText);\n return resultText;\n } catch (error) {\n console.error('Error fetching from Firecrawl:', error);\n throw error;\n }\n};\n\nexport const getContentFromUrl_v1 = async (url: string) => {\n try {\n let ogUrl = url;\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n url = \"https://r.jina.ai/\" + url;\n\n let [response, metadata] = await Promise.all([\n fetch(url),\n getUrlMetadata(ogUrl)\n ]);\n\n const contentTextData = await response.text();\n\n if (!metadata) {\n const html = await getJinaContent(url, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n // console.log('---> metadata?? ', metadata, ' url:' + url)\n let doi\n\n if (metadata) {\n const doiRegex = /10.\\d{4,9}\\/[-._;()\\/:A-Z0-9]+/gi;\n\n\n const doiMatch = JSON.stringify(metadata).match(doiRegex);\n doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n }\n\n console.log('DOI found?:', doi);\n let citation\n\n if (doi) {\n citation = await getCitation(doi);\n } else {\n }\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n let output = \"\"\n if(doi) {\n output = output + \"\\n \" + doi + \"\\n\\n\\n\"\n }\n if(citation) {\n output = output + \"\\n \" + citation + \"\\n\\n\\n\"\n }\n if(contentTextData) {\n output = output + \"\\n \" + contentTextData + \"\\n\\n\\n\"\n }\n if(metadata) {\n output = output + \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\"\n }\n return output\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content: ', error.message || error || 'Unknown error.';\n }\n};\n\nexport const extractDOI = (content) => {\n if(!content) return null;\n if(!content.match) return null;\n \n const doiRegex = DOI_REGEX;\n const doiMatch = content.match(doiRegex);\n let doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n return doi;\n};\n\n\n\n\nexport const getContentFromUrl = async ({ url, optionStr = 'content', crawlMode = 'jina', useCache = true }) => {\n try {\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n let ogUrl = url;\n let urlObject = new URL(url);\n if (urlObject.href.includes('&opts=')) { // Check if the URL contains '&opts='\n // Remove everything from '&opts=' to the end of the string\n // *** this might be brittle!!\n urlObject.href = urlObject.href.split('&opts=')[0];\n }\n\n url = urlObject.toString();\n let str = `${url}_${optionStr}_${crawlMode}`;\n const cacheId = `cache:${stringHash(str)}`;\n\n console.log('[gCFU] useCache:', useCache, 'cacheId:', cacheId)\n\n if (useCache) {\n const cachedContent = await blobby.get(cacheId);\n if (cachedContent) {\n console.log('[gCFU] cache hit:', cacheId);\n return cachedContent;\n }\n }\n\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n const serviceUrl = crawlMode == 'fire' ? url : \"https://r.jina.ai/\" + url;\n\n console.log('getting content from url', serviceUrl);\n\n const optsArray = optionStr.split(',');\n const requestOptions = {\n returnFormat: optsArray.includes('html') ? 'html' : optsArray.includes('json') ? 'json' : undefined,\n withImagesSummary: optsArray.includes('images'),\n withLinksSummary: optsArray.includes('links'),\n withGeneratedAlt: optsArray.includes('caption')\n };\n\n const fetchMetadata = optsArray.includes('metadata') || optsArray.includes('doi') || optsArray.includes('citation');\n let metadata;\n\n if (fetchMetadata) {\n metadata = await getUrlMetadata(ogUrl);\n }\n\n\n const content = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, requestOptions)\n : await getJinaContent(serviceUrl, requestOptions);\n\n if (fetchMetadata && !metadata) {\n const html = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, { returnFormat: 'html' })\n : await getJinaContent(serviceUrl, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n if(!metadata || Object.keys(metadata).length === 0) {\n metadata = content; // for pdfs, it'll just have to be the content\n }\n\n let doi;\n if (optsArray.includes('doi') || optsArray.includes('citation')) {\n if (metadata) {\n doi = extractDOI(JSON.stringify(metadata));\n }\n if (!doi) {\n doi = extractDOI(content);\n }\n if (!doi) {\n doi = extractDOI(metadata);\n }\n\n console.log('DOI found?:', doi);\n }\n\n let citation;\n if (optsArray.includes('citation') && doi) {\n citation = await getCitation(doi);\n }\n\n let output = \"\";\n output += \"\" + serviceUrl + \"\" + \" \";\n output += \"\" + cacheId + \"\";\n output += \"\" + new Date().toLocaleString() + \"\";\n if (optsArray.includes('doi') && doi) {\n output += \"\\n \" + doi + \"\\n\\n\\n\";\n }\n if (optsArray.includes('citation') && citation) {\n output += \"\\n \" + citation + \"\\n\\n\\n\";\n }\n if (optsArray.includes('content') && content) {\n const contentString = typeof content === 'object' ? JSON.stringify(content, null, 2) : content;\n output += \"\\n \" + contentString + \"\\n\\n\\n\";\n }\n if (optsArray.includes('metadata') && metadata) {\n output += \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\";\n }\n\n if (useCache) {\n await blobby.set(cacheId, output);\n console.log('Cache set:', cacheId);\n }\n\n if (output.length === 0) {\n output = '(This area intentionally left blank)';\n }\n\n return output;\n } catch (error) {\n console.error('Error fetching the URL:', error);\n throw new Error('Error fetching the content: ' + error.message || error || 'Unknown error.');\n // return 'Error fetching the content: ' + error.message || error || 'Unknown error.';\n }\n};\n\n\napp.get(\"/*\", async (c) => {\n let url = c.req.query(\"url\");\n let opts = c.req.query(\"opts\") || 'content';\n let crawlMode = c.req.query(\"crawlMode\") || 'jina';\n let noCache = c.req.query(\"noCache\") === 'true';\n let useCache = !noCache;\n\n if (!url) {\n const rawUrl = c.req.raw.url;\n const pathStartIndex = rawUrl.indexOf('/', rawUrl.indexOf('://') + 3);\n url = rawUrl.slice(pathStartIndex + 1); // Get the full path after the initial '/'\n }\n\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n try {\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n return c.text(content);\n } catch (error) {\n return c.text(error.message || error || 'Unknown error.');\n }\n});\n\n\napp.post(\"/\", async (c) => {\n const body = await c.req.json();\n const { url, opts = 'content', crawlMode = 'jina', useCache, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body;\n\n try {\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n \n if (!getSummary && !getTags) {\n return c.text(content);\n }\n \n const [summary, tags, embeddings] = await Promise.all([\n getSummary ? getSummaryFn(getSummaryPrompt, content) : null,\n getTags ? getTagsFn(getTagsPrompt, content) : null,\n getEmbeddings ? getEmbeddingsFn(content) : null\n ]);\n \n return c.json({ summary: summary?.text, tags: tags?.text, content, embeddings });\n } catch (error) {\n return c.json({ error: error.message || error })\n }\n});\n\n\n\n\n\n\n\n\n\n\nexport const getBlurbFromUrl = async (url: string, { content,\n getSummary = true, getSummaryPrompt, getTags = true, getTagsPrompt, getEmbeddings,\n model, provider\n} = {}) => {\n content = content || await getContentFromUrl_v1(url);\n\n if (!getSummary || !getTags) {\n return content;\n }\n\n let summary, tags, embeddings\n if (getSummary) {\n summary = await getSummaryFn(getSummaryPrompt, content, { model, provider });\n }\n if (getTags) {\n tags = await getTagsFn(getTagsPrompt, content, { model, provider });\n }\n if (getEmbeddings) {\n embeddings = await getEmbeddingsFn(content + \" \" + summary + \" \" + tags);\n }\n\n return {\n content,\n summary: summary.text,\n tags: tags.text,\n embeddings\n }\n};\n\nexport const getSummaryFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Summarize this article in 30 words, focusing on the primary author. Do not respond with JSON or XML or stage instructions. Only reply in text. Text:\"\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getTagsFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Return a comma separated list of 3-4 tags that broadly describes the text. Do not repeat the given text. Just return text and commas, no other symbols. Text to categorize:\"\n console.log(\"prompt\", prompt)\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getEmbeddingsFn = async (content) => {\n let result = await ai({\n provider: \"openai\",\n embed: true,\n value: content\n })\n return result\n}\n\n\n\n\n\nexport default app.fetch;\n","/** @jsx jsx */\nimport { jsx } from \"https://deno.land/x/hono@v3.11.7/middleware.ts\";\nimport { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { ai } from \"https://esm.town/v/yawnxyz/ai\";\nimport { getUrlMetadata } from \"https://esm.town/v/yawnxyz/urlMetadata\";\nimport { getHtmlMetadata } from \"https://esm.town/v/yawnxyz/htmlMetadata\";\nimport { getCitation } from \"https://esm.town/v/yawnxyz/citation\";\nimport { blobby } from \"https://esm.town/v/@yawnxyz/blobby\";\nimport stringHash from 'npm:string-hash';\n\nconst app = new Hono();\n\n// https://www.crossref.org/blog/dois-and-matching-regular-expressions/\nconst DOI_REGEX = /\\b(10\\.\\d{4,9}\\/[-._;()\\/:\\w]+)\\b/i;\n\nexport const getJinaContent = async (url, opts = {}) => {\n const baseUrl = 'https://r.jina.ai/';\n if (!url.includes('r.jina.ai')) {\n url = baseUrl + url;\n }\n const fullUrl = new URL(url);\n\n const headers = {\n ...(opts.withImagesSummary && { 'X-With-Images-Summary': 'true' }),\n ...(opts.withGeneratedAlt && { 'X-With-Generated-Alt': 'true' }),\n ...(opts.withLinksSummary && { 'X-With-Links-Summary': 'true' }),\n ...(opts.noCache && { 'X-No-Cache': 'true' }),\n ...(opts.accept && { 'Accept': opts.accept }),\n ...(opts.targetSelector && { 'X-Target-Selector': opts.targetSelector }),\n ...(opts.timeout && { 'X-Timeout': opts.timeout.toString() }),\n ...(opts.waitForSelector && { 'X-Wait-For-Selector': opts.waitForSelector }),\n ...(opts.returnFormat && { 'X-Return-Format': opts.returnFormat }),\n };\n\n console.log('[getJinaContent] Fetching:', fullUrl.toString(), headers);\n\n try {\n const response = await fetch(fullUrl.toString(), {\n method: 'GET',\n headers: headers,\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n if (headers['Accept'] === 'text/event-stream') {\n return response.body;\n }\n\n const text = await response.text();\n try {\n return JSON.parse(text);\n } catch {\n if (headers['X-Return-Format'] === 'screenshot') {\n return JSON.parse(text);\n } else {\n return text;\n }\n }\n } catch (error) {\n console.error('Error fetching from Jina:', error);\n throw error;\n }\n}\n\nexport const getFirecrawlContent = async (url, opts = {}) => {\n const apiKey = Deno.env.get(\"FIRECRAWL_API_KEY\");\n if (!apiKey) {\n throw new Error(\"API key not found. Please set FIRECRAWL_API_KEY in your environment.\");\n }\n\n const body = {\n url: url,\n pageOptions: {\n onlyMainContent: opts.onlyMainContent || true,\n includeHtml: opts.includeHtml || false,\n includeRawHtml: opts.includeRawHtml || false,\n screenshot: opts.screenshot || false,\n waitFor: opts.waitFor || 0,\n removeTags: opts.removeTags || [],\n onlyIncludeTags: opts.onlyIncludeTags || [],\n replaceAllPathsWithAbsolutePaths: opts.replaceAllPathsWithAbsolutePaths || true,\n parsePDF: opts.parsePDF || false\n },\n extractorOptions: {\n // mode: opts.mode || \"llm-extraction\",\n mode: opts.mode,\n extractionPrompt: opts.extractionPrompt || \"\",\n extractionSchema: opts.extractionSchema || {}\n },\n timeout: opts.timeout || 30000\n };\n\n console.log('[getFirecrawlContent] Fetching:', url, body);\n\n try {\n const response = await fetch(\"https://api.firecrawl.dev/v0/scrape\", {\n method: 'POST',\n headers: {\n \"Authorization\": `Bearer ${apiKey}`,\n \"Content-Type\": \"application/json\"\n },\n body: JSON.stringify(body)\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const result = await response.json();\n let resultText = JSON.stringify(result);\n console.log('[getFirecrawlContent] Result:', resultText);\n return resultText;\n } catch (error) {\n console.error('Error fetching from Firecrawl:', error);\n throw error;\n }\n};\n\nexport const getContentFromUrl_v1 = async (url: string) => {\n try {\n let ogUrl = url;\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n url = \"https://r.jina.ai/\" + url;\n\n let [response, metadata] = await Promise.all([\n fetch(url),\n getUrlMetadata(ogUrl)\n ]);\n\n const contentTextData = await response.text();\n\n if (!metadata) {\n const html = await getJinaContent(url, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n // console.log('---> metadata?? ', metadata, ' url:' + url)\n let doi\n\n if (metadata) {\n const doiRegex = /10.\\d{4,9}\\/[-._;()\\/:A-Z0-9]+/gi;\n\n\n const doiMatch = JSON.stringify(metadata).match(doiRegex);\n doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n }\n\n console.log('DOI found?:', doi);\n let citation\n\n if (doi) {\n citation = await getCitation(doi);\n } else {\n }\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n let output = \"\"\n if(doi) {\n output = output + \"\\n \" + doi + \"\\n\\n\\n\"\n }\n if(citation) {\n output = output + \"\\n \" + citation + \"\\n\\n\\n\"\n }\n if(contentTextData) {\n output = output + \"\\n \" + contentTextData + \"\\n\\n\\n\"\n }\n if(metadata) {\n output = output + \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\"\n }\n return output\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content: ', error.message || error || 'Unknown error.';\n }\n};\n\nexport const extractDOI = (content) => {\n if(!content) return null;\n if(!content.match) return null;\n \n const doiRegex = DOI_REGEX;\n const doiMatch = content.match(doiRegex);\n let doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n return doi;\n};\n\n\n\n\nexport const getContentFromUrl = async ({ url, optionStr = 'content', crawlMode = 'jina', useCache = true }) => {\n try {\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n let ogUrl = url;\n let urlObject = new URL(url);\n if (urlObject.href.includes('&opts=')) { // Check if the URL contains '&opts='\n // Remove everything from '&opts=' to the end of the string\n // *** this might be brittle!!\n urlObject.href = urlObject.href.split('&opts=')[0];\n }\n\n url = urlObject.toString();\n let str = `${url}_${optionStr}_${crawlMode}`;\n const cacheId = `cache:${stringHash(str)}`;\n\n console.log('[gCFU] useCache:', useCache, 'cacheId:', cacheId)\n\n if (useCache) {\n const cachedContent = await blobby.get(cacheId);\n if (cachedContent) {\n console.log('[gCFU] cache hit:', cacheId);\n return cachedContent;\n }\n }\n\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n const serviceUrl = crawlMode == 'fire' ? url : \"https://r.jina.ai/\" + url;\n\n console.log('getting content from url', serviceUrl);\n\n const optsArray = optionStr.split(',');\n const requestOptions = {\n returnFormat: optsArray.includes('html') ? 'html' : optsArray.includes('json') ? 'json' : undefined,\n withImagesSummary: optsArray.includes('images'),\n withLinksSummary: optsArray.includes('links'),\n withGeneratedAlt: optsArray.includes('caption')\n };\n\n const fetchMetadata = optsArray.includes('metadata') || optsArray.includes('doi') || optsArray.includes('citation');\n let metadata;\n\n if (fetchMetadata) {\n metadata = await getUrlMetadata(ogUrl);\n }\n\n\n const content = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, requestOptions)\n : await getJinaContent(serviceUrl, requestOptions);\n\n if (fetchMetadata && !metadata) {\n const html = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, { returnFormat: 'html' })\n : await getJinaContent(serviceUrl, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n if(!metadata || Object.keys(metadata).length === 0) {\n metadata = content; // for pdfs, it'll just have to be the content\n }\n\n let doi;\n if (optsArray.includes('doi') || optsArray.includes('citation')) {\n if (metadata) {\n doi = extractDOI(JSON.stringify(metadata));\n }\n if (!doi) {\n doi = extractDOI(content);\n }\n if (!doi) {\n doi = extractDOI(metadata);\n }\n\n console.log('DOI found?:', doi);\n }\n\n let citation;\n if (optsArray.includes('citation') && doi) {\n citation = await getCitation(doi);\n }\n\n let output = \"\";\n output += \"\" + serviceUrl + \"\" + \" \";\n output += \"\" + cacheId + \"\";\n output += \"\" + new Date().toLocaleString() + \"\";\n if (optsArray.includes('doi') && doi) {\n output += \"\\n \" + doi + \"\\n\\n\\n\";\n }\n if (optsArray.includes('citation') && citation) {\n output += \"\\n \" + citation + \"\\n\\n\\n\";\n }\n if (optsArray.includes('content') && content) {\n const contentString = typeof content === 'object' ? JSON.stringify(content, null, 2) : content;\n output += \"\\n \" + contentString + \"\\n\\n\\n\";\n }\n if (optsArray.includes('metadata') && metadata) {\n output += \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\";\n }\n\n if (useCache) {\n await blobby.set(cacheId, output);\n console.log('Cache set:', cacheId);\n }\n\n if (output.length === 0) {\n output = '(This area intentionally left blank)';\n }\n\n return output;\n } catch (error) {\n console.error('Error fetching the URL:', error);\n throw new Error('Error fetching the content: ' + error.message || error || 'Unknown error.');\n // return 'Error fetching the content: ' + error.message || error || 'Unknown error.';\n }\n};\n\n\napp.get(\"/*\", async (c) => {\n let url = c.req.query(\"url\");\n let opts = c.req.query(\"opts\") || 'content';\n let crawlMode = c.req.query(\"crawlMode\") || 'jina';\n let noCache = c.req.query(\"noCache\") === 'true';\n let useCache = !noCache;\n\n if (!url) {\n const rawUrl = c.req.raw.url;\n const pathStartIndex = rawUrl.indexOf('/', rawUrl.indexOf('://') + 3);\n url = rawUrl.slice(pathStartIndex + 1); // Get the full path after the initial '/'\n }\n\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n try {\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n return c.text(content);\n } catch (error) {\n return c.text(error.message || error || 'Unknown error.');\n }\n});\n\n\napp.post(\"/\", async (c) => {\n const body = await c.req.json();\n const { url, opts = 'content', crawlMode = 'jina', useCache, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body;\n\n try {\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n \n if (!getSummary && !getTags) {\n return c.text(content);\n }\n \n const [summary, tags, embeddings] = await Promise.all([\n getSummary ? getSummaryFn(getSummaryPrompt, content) : null,\n getTags ? getTagsFn(getTagsPrompt, content) : null,\n getEmbeddings ? getEmbeddingsFn(content) : null\n ]);\n \n return c.json({ summary: summary?.text, tags: tags?.text, content, embeddings });\n } catch (error) {\n return c.json({ error: error.message || error || \"Unknown error.\"})\n }\n});\n\n\n\n\n\n\n\n\n\n\nexport const getBlurbFromUrl = async (url: string, { content,\n getSummary = true, getSummaryPrompt, getTags = true, getTagsPrompt, getEmbeddings,\n model, provider\n} = {}) => {\n content = content || await getContentFromUrl_v1(url);\n\n if (!getSummary || !getTags) {\n return content;\n }\n\n let summary, tags, embeddings\n if (getSummary) {\n summary = await getSummaryFn(getSummaryPrompt, content, { model, provider });\n }\n if (getTags) {\n tags = await getTagsFn(getTagsPrompt, content, { model, provider });\n }\n if (getEmbeddings) {\n embeddings = await getEmbeddingsFn(content + \" \" + summary + \" \" + tags);\n }\n\n return {\n content,\n summary: summary.text,\n tags: tags.text,\n embeddings\n }\n};\n\nexport const getSummaryFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Summarize this article in 30 words, focusing on the primary author. Do not respond with JSON or XML or stage instructions. Only reply in text. Text:\"\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getTagsFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Return a comma separated list of 3-4 tags that broadly describes the text. Do not repeat the given text. Just return text and commas, no other symbols. Text to categorize:\"\n console.log(\"prompt\", prompt)\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getEmbeddingsFn = async (content) => {\n let result = await ai({\n provider: \"openai\",\n embed: true,\n value: content\n })\n return result\n}\n\n\n\n\n\nexport default app.fetch;\n","createdAt",["D",1722524145670],"branchVersion","branchId","0b9c2cd4-2793-11f0-9d52-569c3dd06744","id","0b9beaa8-2793-11f0-9d52-569c3dd06744","getContentFromUrl",["D",1746217295690],"deletedAt","legacyEnvVars",true,"privacy","public","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",1,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_93":92,"_98":95},"org",{"_93":92,"_94":95,"_18":53,"_96":97},"branch",{"_13":82,"_31":83,"_42":41,"_84":43,"_85":-5,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":44,"_91":92},"isOwned",256,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","profileImageUrl","branchName","",{"_42":43,"_31":44,"_37":151,"_46":-5,"_47":48,"_49":50,"_51":-5,"_52":-5,"_18":53,"_54":30,"_55":48,"_56":-5,"_57":-5,"_58":59,"_60":61,"_62":-5,"_63":-5,"_64":-5,"_65":-5,"_66":67,"_68":67,"_69":70,"_71":70,"_72":70,"_73":67,"_74":67,"_75":152,"_77":153,"_79":154,"_81":25},"tree",[142,143],"files",[142,143],"counts",{"_132":129,"_133":59,"_134":59,"_135":136,"_137":138},"contents",[116,117],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"navCollapsed",{"_42":125,"_31":126,"_29":127,"_119":-5,"_40":41,"_39":59,"_120":48,"_37":128,"_13":129,"_32":20,"_34":130,"_24":25,"_123":-5,"_26":131},{"_42":118,"_31":27,"_29":30,"_119":-5,"_40":41,"_39":14,"_120":25,"_37":121,"_13":122,"_32":33,"_34":36,"_24":25,"_123":-5,"_26":124},"111647ee-2793-11f0-9d52-569c3dd06744","parentId","latest",["D",1722524145670],239,"label",[27],"0b9eaa5e-2793-11f0-9d52-569c3dd06744","README.md","0b9eaacc-2793-11f0-9d52-569c3dd06744",["D",1718163352826],0,"## getContentFromUrl\nUse this for summarizers.\nCombines `https://r.jina.ai/URL` and markdown.download's Youtube transcription getter to do its best to retrieve content from URLs.\n\nhttps://arstechnica.com/space/2024/06/nasa-indefinitely-delays-return-of-starliner-to-review-propulsion-data \nhttps://journals.asm.org/doi/10.1128/iai.00065-23\n\nUsage:\n- https://yawnxyz-getcontentfromurl.web.val.run/https://www.ncbi.nlm.nih.gov/pmc/articles/PMC10187409/\n- https://yawnxyz-getcontentfromurl.web.val.run/https://www.youtube.com/watch?v=gzsczZnS84Y&ab_channel=PhageDirectory\n\nMigrated from folder: Libraries/scrapers/getContentFromUrl/getContentFromUrl",[126],"pulls","branches","forks","envVars",{"_139":129,"_141":129},"connections",{"_139":129,"_140":129},"count","unfilledCount","emptyCount",{"_42":125,"_31":126,"_29":127,"_119":-5,"_40":41,"_39":59,"_120":48,"_37":148,"_13":129,"_32":20,"_34":130,"_24":25,"_123":-5,"_145":149,"_26":150},{"_42":118,"_31":27,"_29":30,"_119":-5,"_40":41,"_39":14,"_120":25,"_37":144,"_13":122,"_32":33,"_34":36,"_24":25,"_123":-5,"_145":146,"_26":147},["D",1722524145670],"children",[],[27],["D",1718163352826],[],[126],["D",1746217295690],{"_93":92,"_98":95},{"_93":92,"_94":95,"_18":53,"_96":97},{"_13":82,"_31":83,"_42":41,"_84":43,"_85":-5,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":44,"_91":92},"ctx",{"_75":-5,"_188":189,"_190":191,"_192":25,"_193":25,"_194":25,"_195":-5,"_196":-7,"_197":-7,"_198":129,"_199":25,"_200":25,"_201":-5,"_202":25,"_203":25},"ENV",{"_159":160,"_161":162,"_163":164,"_165":83,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","f1316bb1ce3ad41ce227a17bcad7725ad044419f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","API_URL","https://api.val.town","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier"]

Diff · v240