[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_155":156,"_157":158},"routes/_app.x.$handle.$name",{"_16":17,"_20":21,"_83":66,"_84":85,"_64":-7,"_86":-5,"_13":87,"_88":89,"_11":90,"_91":92,"_93":94,"_95":-5,"_96":-5,"_97":-5,"_98":63,"_99":100,"_101":26,"_102":-5,"_103":-5,"_104":105},"routes/_app.x.$handle.$name.search.$(branch)",{"_11":12,"_13":14,"_15":-5,"_16":17},"files",[],"project",{"_18":19,"_20":21,"_22":23,"_24":-5,"_25":26,"_27":28,"_29":28,"_30":-5,"_31":-5,"_32":33,"_34":35,"_36":26,"_37":-5,"_38":-5,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":-5,"_46":-5,"_47":48,"_49":48,"_50":51,"_52":51,"_53":51,"_54":48,"_55":48,"_56":57,"_58":59,"_60":61,"_62":63},"query","handle","yawnxyz","id","0b9beaa8-2793-11f0-9d52-569c3dd06744","name","getContentFromUrl","createdAt",["D",1746217295690],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","userId","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","0b9ea5ea-2793-11f0-9d52-569c3dd06744","pinPreview","pinMigrated","screenshotVersion","likeCount",1,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_16":17,"_82":78},"org",{"_16":17,"_76":-5,"_77":78,"_32":33,"_79":80,"_81":63},"branch",{"_64":65,"_20":66,"_18":67,"_68":19,"_69":-5,"_70":-5,"_71":-5,"_72":-5,"_73":-5,"_74":21,"_75":17},"isOwned",false,"version",256,"main","0b9c2cd4-2793-11f0-9d52-569c3dd06744","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","path","","adoptOrgId",{"_18":19,"_20":21,"_22":151,"_24":-5,"_25":26,"_27":28,"_29":28,"_30":-5,"_31":-5,"_32":33,"_34":35,"_36":26,"_37":-5,"_38":-5,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":-5,"_46":-5,"_47":48,"_49":48,"_50":51,"_52":51,"_53":51,"_54":48,"_55":48,"_56":152,"_58":153,"_60":154,"_62":63},"tree",[142,143],[142,143],"counts",{"_132":129,"_133":40,"_134":40,"_135":136,"_137":138},"contents",[107,108],"file","upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_35":106},"content.coversheet.org",{"_18":125,"_20":126,"_111":127,"_112":-5,"_113":67,"_114":40,"_115":26,"_22":128,"_64":129,"_118":95,"_120":130,"_122":63,"_123":-5,"_84":131},{"_18":109,"_20":110,"_111":35,"_112":-5,"_113":67,"_114":65,"_115":26,"_22":116,"_64":117,"_118":119,"_120":121,"_122":63,"_123":-5,"_84":124},"1191683e-2793-11f0-9d52-569c3dd06744","main.tsx","fileId","parentId","branchId","branchVersion","latest",["D",1724515929509],255,"fileType","http","content","/** @jsx jsx */\nimport { jsx } from \"https://deno.land/x/hono@v3.11.7/middleware.ts\";\nimport { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { ai } from \"https://esm.town/v/yawnxyz/ai\";\nimport { getUrlMetadata } from \"https://esm.town/v/yawnxyz/urlMetadata\";\nimport { getHtmlMetadata } from \"https://esm.town/v/yawnxyz/htmlMetadata\";\nimport { getCitation } from \"https://esm.town/v/yawnxyz/citation\";\nimport { blobby } from \"https://esm.town/v/yawnxyz/blobby\";\nimport { transcribeAudio } from \"https://esm.town/v/yawnxyz/stt\";\nimport stringHash from 'npm:string-hash';\n\nconst app = new Hono();\n\n// https://www.crossref.org/blog/dois-and-matching-regular-expressions/\nconst DOI_REGEX = /\\b(10\\.\\d{4,9}\\/[-._;()\\/:\\w]+)\\b/i;\n\nexport const getJinaContent = async (url, opts = {}) => {\n const baseUrl = 'https://r.jina.ai/';\n if (!url.includes('r.jina.ai')) {\n url = baseUrl + url;\n }\n const fullUrl = new URL(url);\n\n const headers = {\n ...(opts.withImagesSummary && { 'X-With-Images-Summary': 'true' }),\n ...(opts.withGeneratedAlt && { 'X-With-Generated-Alt': 'true' }),\n ...(opts.withLinksSummary && { 'X-With-Links-Summary': 'true' }),\n ...(opts.noCache && { 'X-No-Cache': 'true' }),\n ...(opts.accept && { 'Accept': opts.accept }),\n ...(opts.targetSelector && { 'X-Target-Selector': opts.targetSelector }),\n ...(opts.timeout && { 'X-Timeout': opts.timeout.toString() }),\n ...(opts.waitForSelector && { 'X-Wait-For-Selector': opts.waitForSelector }),\n ...(opts.returnFormat && { 'X-Return-Format': opts.returnFormat }),\n };\n\n console.log('[getJinaContent] Fetching:', fullUrl.toString(), headers);\n\n try {\n const response = await fetch(fullUrl.toString(), {\n method: 'GET',\n headers: headers,\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n if (headers['Accept'] === 'text/event-stream') {\n return response.body;\n }\n\n const text = await response.text();\n try {\n return JSON.parse(text);\n } catch {\n if (headers['X-Return-Format'] === 'screenshot') {\n return JSON.parse(text);\n } else {\n return text;\n }\n }\n } catch (error) {\n console.error('Error fetching from Jina:', error);\n throw error;\n }\n}\n\nexport const getFirecrawlContent = async (url, opts = {}) => {\n const apiKey = Deno.env.get(\"FIRECRAWL_API_KEY\");\n if (!apiKey) {\n throw new Error(\"API key not found. Please set FIRECRAWL_API_KEY in your environment.\");\n }\n\n const body = {\n url: url,\n pageOptions: {\n onlyMainContent: opts.onlyMainContent || true,\n includeHtml: opts.includeHtml || false,\n includeRawHtml: opts.includeRawHtml || false,\n screenshot: opts.screenshot || false,\n waitFor: opts.waitFor || 0,\n removeTags: opts.removeTags || [],\n onlyIncludeTags: opts.onlyIncludeTags || [],\n replaceAllPathsWithAbsolutePaths: opts.replaceAllPathsWithAbsolutePaths || true,\n parsePDF: opts.parsePDF || false\n },\n extractorOptions: {\n // mode: opts.mode || \"llm-extraction\",\n mode: opts.mode,\n extractionPrompt: opts.extractionPrompt || \"\",\n extractionSchema: opts.extractionSchema || {}\n },\n timeout: opts.timeout || 30000\n };\n\n console.log('[getFirecrawlContent] Fetching:', url, body);\n\n try {\n const response = await fetch(\"https://api.firecrawl.dev/v0/scrape\", {\n method: 'POST',\n headers: {\n \"Authorization\": `Bearer ${apiKey}`,\n \"Content-Type\": \"application/json\"\n },\n body: JSON.stringify(body)\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const result = await response.json();\n let resultText = JSON.stringify(result);\n console.log('[getFirecrawlContent] Result:', resultText);\n return resultText;\n } catch (error) {\n console.error('Error fetching from Firecrawl:', error);\n throw error;\n }\n};\n\nexport const getContentFromUrl_v1 = async (url: string) => {\n try {\n let ogUrl = url;\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n url = \"https://r.jina.ai/\" + url;\n\n let [response, metadata] = await Promise.all([\n fetch(url),\n getUrlMetadata(ogUrl)\n ]);\n\n const contentTextData = await response.text();\n\n if (!metadata) {\n const html = await getJinaContent(url, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n // console.log('---> metadata?? ', metadata, ' url:' + url)\n let doi\n\n if (metadata) {\n const doiRegex = /10.\\d{4,9}\\/[-._;()\\/:A-Z0-9]+/gi;\n\n\n const doiMatch = JSON.stringify(metadata).match(doiRegex);\n doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n }\n\n console.log('DOI found?:', doi);\n let citation\n\n if (doi) {\n citation = await getCitation(doi);\n } else {\n }\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n let output = \"\"\n if(doi) {\n output = output + \"\\n \" + doi + \"\\n\\n\\n\"\n }\n if(citation) {\n output = output + \"\\n \" + citation + \"\\n\\n\\n\"\n }\n if(contentTextData) {\n output = output + \"\\n \" + contentTextData + \"\\n\\n\\n\"\n }\n if(metadata) {\n output = output + \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\"\n }\n return output\n } catch (error) {\n console.error('Error fetching the URL:', error);\n return 'Error fetching the content: ', error.message || error || 'Unknown error.';\n }\n};\n\nexport const extractDOI = (content) => {\n if(!content) return null;\n if(!content.match) return null;\n \n const doiRegex = DOI_REGEX;\n const doiMatch = content.match(doiRegex);\n let doi = doiMatch ? doiMatch[0] : null;\n if (doi) {\n doi = doi.replace(/^doi[:\\/]/i, '');\n }\n return doi;\n};\n\n\n\n\nexport const getContentFromUrl = async ({ url, optionStr = 'content', crawlMode = 'jina', useCache = true }) => {\n try {\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n // Check if the URL is an MP3 file\n if (url.endsWith('.mp3')) {\n const transcript = await transcribeAudio(url); // Get transcript for MP3\n const output = `\\n ${transcript}\\n\\n\\n`;\n const cacheId = `cache:${stringHash(`${url}_content_mp3`)}`; // Create a cache ID for the MP3\n if (useCache) {\n await blobby.set(cacheId, output); // Cache the output\n console.log('Cache set:', cacheId);\n }\n return `${url} ${cacheId}${output}`; // Return source and cache ID\n }\n\n\n let ogUrl = url;\n let urlObject = new URL(url);\n if (urlObject.href.includes('&opts=')) { // Check if the URL contains '&opts='\n // Remove everything from '&opts=' to the end of the string\n // *** this might be brittle!!\n urlObject.href = urlObject.href.split('&opts=')[0];\n }\n\n url = urlObject.toString();\n let str = `${url}_${optionStr}_${crawlMode}`;\n const cacheId = `cache:${stringHash(str)}`;\n\n console.log('[gCFU] useCache:', useCache, 'cacheId:', cacheId)\n\n if (useCache) {\n const cachedContent = await blobby.get(cacheId);\n if (cachedContent) {\n console.log('[gCFU] cache hit:', cacheId);\n return cachedContent;\n }\n }\n\n if (url.includes(\"youtube\") || url.includes(\"youtu.be\")) {\n url = `https://val.markdown.download/?url=${url}`;\n }\n const serviceUrl = crawlMode == 'fire' ? url : \"https://r.jina.ai/\" + url;\n\n console.log('getting content from url', serviceUrl);\n\n const optsArray = optionStr.split(',');\n const requestOptions = {\n returnFormat: optsArray.includes('html') ? 'html' : optsArray.includes('json') ? 'json' : undefined,\n withImagesSummary: optsArray.includes('images'),\n withLinksSummary: optsArray.includes('links'),\n withGeneratedAlt: optsArray.includes('caption')\n };\n\n const fetchMetadata = optsArray.includes('metadata') || optsArray.includes('doi') || optsArray.includes('citation');\n let metadata;\n\n if (fetchMetadata) {\n metadata = await getUrlMetadata(ogUrl);\n }\n\n\n const content = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, requestOptions)\n : await getJinaContent(serviceUrl, requestOptions);\n\n if (fetchMetadata && !metadata) {\n const html = crawlMode === 'fire'\n ? await getFirecrawlContent(serviceUrl, { returnFormat: 'html' })\n : await getJinaContent(serviceUrl, { returnFormat: 'html' });\n metadata = await getHtmlMetadata(html);\n }\n\n\n if(!metadata || Object.keys(metadata).length === 0) {\n metadata = content; // for pdfs, it'll just have to be the content\n }\n\n let doi;\n if (optsArray.includes('doi') || optsArray.includes('citation')) {\n if (metadata) {\n doi = extractDOI(JSON.stringify(metadata));\n }\n if (!doi) {\n doi = extractDOI(content);\n }\n if (!doi) {\n doi = extractDOI(metadata);\n }\n\n console.log('DOI found?:', doi);\n }\n\n let citation;\n if (optsArray.includes('citation') && doi) {\n citation = await getCitation(doi);\n }\n\n let output = \"\";\n output += \"\" + serviceUrl + \"\" + \" \";\n output += \"\" + cacheId + \"\";\n output += \"\" + new Date().toLocaleString() + \"\";\n if (optsArray.includes('doi') && doi) {\n output += \"\\n \" + doi + \"\\n\\n\\n\";\n }\n if (optsArray.includes('citation') && citation) {\n output += \"\\n \" + citation + \"\\n\\n\\n\";\n }\n if ((optsArray.includes('content')||optsArray.includes('html')) && content) {\n const contentString = typeof content === 'object' ? JSON.stringify(content, null, 2) : content;\n output += \"\\n \" + contentString + \"\\n\\n\\n\";\n }\n if (optsArray.includes('metadata') && metadata) {\n output += \"\\n \" + JSON.stringify(metadata, null, 2) + \"\\n\\n\\n\";\n }\n\n if (useCache) {\n await blobby.set(cacheId, output);\n console.log('Cache set:', cacheId);\n }\n\n if (output.length === 0) {\n output = '(This area intentionally left blank)';\n }\n\n return output;\n } catch (error) {\n console.error('Error fetching the URL:', error);\n throw new Error('Error fetching the content: ' + error.message || error || 'Unknown error.');\n // return 'Error fetching the content: ' + error.message || error || 'Unknown error.';\n }\n};\n\n\napp.get(\"/*\", async (c) => {\n let url = c.req.query(\"url\");\n let opts = c.req.query(\"opts\") || 'content';\n let crawlMode = c.req.query(\"crawlMode\") || 'jina';\n let useCache = typeof c.req.query(\"useCache\") !== 'undefined' ? c.req.query(\"useCache\") : true;\n // let noCache = c.req.query(\"noCache\") === 'true';\n // let useCache = !noCache;\n\n if (!url) {\n const rawUrl = c.req.raw.url;\n const pathStartIndex = rawUrl.indexOf('/', rawUrl.indexOf('://') + 3);\n url = rawUrl.slice(pathStartIndex + 1); // Get the full path after the initial '/'\n }\n\n // Check if the input is a pure DOI\n const doiRegex = DOI_REGEX;\n if (doiRegex.test(url) && !url.includes(\"://\")) {\n url = `https://doi.org/${url}`;\n } else if (!url.startsWith(\"http\")) {\n return c.text(\"URL query parameter, valid path, or DOI is required.\");\n }\n\n try {\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n return c.text(content);\n } catch (error) {\n return c.text(error.message || error || 'Unknown error.');\n }\n});\n\n\napp.post(\"/\", async (c) => {\n const body = await c.req.json();\n const { url, opts = 'content', crawlMode = 'jina', useCache, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body;\n\n try {\n const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache });\n \n if (!getSummary && !getTags) {\n return c.text(content);\n }\n \n const [summary, tags, embeddings] = await Promise.all([\n getSummary ? getSummaryFn(getSummaryPrompt, content) : null,\n getTags ? getTagsFn(getTagsPrompt, content) : null,\n getEmbeddings ? getEmbeddingsFn(content) : null\n ]);\n \n return c.json({ summary: summary?.text, tags: tags?.text, content, embeddings });\n } catch (error) {\n return c.json({ error: error.message || error || \"Unknown error.\"})\n }\n});\n\n\n\n\n\n\n\n\n\n\nexport const getBlurbFromUrl = async (url: string, { content,\n getSummary = true, getSummaryPrompt, getTags = true, getTagsPrompt, getEmbeddings,\n model, provider\n} = {}) => {\n content = content || await getContentFromUrl_v1(url);\n\n if (!getSummary || !getTags) {\n return content;\n }\n\n let summary, tags, embeddings\n if (getSummary) {\n summary = await getSummaryFn(getSummaryPrompt, content, { model, provider });\n }\n if (getTags) {\n tags = await getTagsFn(getTagsPrompt, content, { model, provider });\n }\n if (getEmbeddings) {\n embeddings = await getEmbeddingsFn(content + \" \" + summary + \" \" + tags);\n }\n\n return {\n content,\n summary: summary.text,\n tags: tags.text,\n embeddings\n }\n};\n\nexport const getSummaryFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Summarize this article in 30 words, focusing on the primary author. Do not respond with JSON or XML or stage instructions. Only reply in text. Text:\"\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getTagsFn = async (prompt, content, {\n provider,\n model,\n} = {}) => {\n prompt = prompt || \"Return a comma separated list of 3-4 tags that broadly describes the text. Do not repeat the given text. Just return text and commas, no other symbols. Text to categorize:\"\n console.log(\"prompt\", prompt)\n let result = await ai({\n provider: provider || \"openai\",\n model: model || \"gpt-3.5-turbo\",\n prompt: prompt + content\n })\n return result\n}\n\nexport const getEmbeddingsFn = async (content) => {\n let result = await ai({\n provider: \"openai\",\n embed: true,\n value: content\n })\n return result\n}\n\n\n\n\n\nexport default app.fetch;\n","deleted","label",[110],"0b9eaa5e-2793-11f0-9d52-569c3dd06744","README.md","0b9eaacc-2793-11f0-9d52-569c3dd06744",["D",1718163352826],0,"## getContentFromUrl\nUse this for summarizers.\nCombines `https://r.jina.ai/URL` and markdown.download's Youtube transcription getter to do its best to retrieve content from URLs.\n\nhttps://arstechnica.com/space/2024/06/nasa-indefinitely-delays-return-of-starliner-to-review-propulsion-data \nhttps://journals.asm.org/doi/10.1128/iai.00065-23\n\nUsage:\n- https://yawnxyz-getcontentfromurl.web.val.run/https://www.ncbi.nlm.nih.gov/pmc/articles/PMC10187409/\n- https://yawnxyz-getcontentfromurl.web.val.run/https://www.youtube.com/watch?v=gzsczZnS84Y&ab_channel=PhageDirectory\n\nMigrated from folder: Libraries/scrapers/getContentFromUrl/getContentFromUrl",[126],"pulls","branches","forks","envVars",{"_139":129,"_141":129},"connections",{"_139":129,"_140":129},"count","unfilledCount","emptyCount",{"_18":125,"_20":126,"_111":127,"_112":-5,"_113":67,"_114":40,"_115":26,"_22":148,"_64":129,"_118":95,"_120":130,"_122":63,"_123":-5,"_145":149,"_84":150},{"_18":109,"_20":110,"_111":35,"_112":-5,"_113":67,"_114":65,"_115":26,"_22":144,"_64":117,"_118":119,"_120":121,"_122":63,"_123":-5,"_145":146,"_84":147},["D",1724515929509],"children",[],[110],["D",1718163352826],[],[126],["D",1746217295690],{"_16":17,"_82":78},{"_16":17,"_76":-5,"_77":78,"_32":33,"_79":80,"_81":63},{"_64":65,"_20":66,"_18":67,"_68":19,"_69":-5,"_70":-5,"_71":-5,"_72":-5,"_73":-5,"_74":21,"_75":17},"ctx",{"_56":-5,"_103":188,"_189":190,"_191":192,"_193":63,"_194":63,"_195":63,"_196":26,"_197":-5,"_198":-7,"_199":-7,"_200":129,"_201":63,"_202":63,"_203":-5,"_204":63,"_205":63,"_206":63,"_207":-5},"ENV",{"_159":160,"_161":162,"_163":66,"_164":165,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","c73f21da490985bff19c279eb0e09db24d688e4f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Search