[{"_1":2,"_182":-5,"_183":-5},"loaderData",{"_3":4,"_95":96},"root",{"_5":6,"_36":37,"_68":69},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isBusinessOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","f33c6f02e3b9304fe5cb194294cedf6a64b697a5","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","API_URL","https://api.val.town","clerkState",{"_70":71},"__internal_clerk_state",{"_72":73,"_74":47,"_75":76,"_77":76,"_78":20,"_79":76,"_80":76,"_81":76,"_82":76,"_83":84,"_93":20,"_94":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_85":20,"_19":20,"_86":76,"_87":88,"_89":76,"_90":47,"_91":20,"_92":76},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_97":98,"_99":100,"_101":102,"_103":104,"_156":157,"_178":179,"_180":181},"handle","Tamas_gr","name","ai-feed","path","src/lib/extract.ts","project",{"_105":106,"_99":100,"_107":108,"_109":-5,"_110":20,"_111":112,"_113":76,"_114":-5,"_12":115,"_116":-5,"_117":20,"_118":-5,"_119":-5,"_120":33,"_14":121,"_122":-5,"_123":-5,"_124":-5,"_125":126,"_127":126,"_128":129,"_130":129,"_131":129,"_132":126,"_133":126,"_23":134,"_137":138,"_142":143,"_155":20},"id","6d5266d2-2c0b-11f1-aabf-42dde27851f2","createdAt",["D",1774856301553],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","019d3d8a-7c54-7244-9df3-c7566f8043d2","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","019d3d8a-7c54-7244-9df3-cb76fe1ca972","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_97":98,"_135":136},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18zQmVlU3pHM2tJT1VyUTZBU1lPNHA4aExLMEUifQ","org",{"_97":98,"_139":136,"_12":115,"_140":141},"avatar","tier","free","branch",{"_144":145,"_99":45,"_105":146,"_147":106,"_148":-5,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":100,"_154":98},"version",9,"6d52cd20-2c0b-11f1-aabf-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_105":158,"_99":159,"_160":161,"_162":163,"_164":146,"_165":166,"_167":168,"_107":169,"_144":33,"_170":156,"_171":172,"_173":20,"_174":-5,"_101":175},"305e4fa0-2c0d-11f1-9af2-42dde27851f2","extract.ts","fileId","305e5036-2c0d-11f1-9af2-42dde27851f2","parentId","f05af2f7-155e-4d94-962c-00472e65160e","branchId","branchVersion",3,"latest",true,["D",1774857058282],"fileType","content","import type { ExtractedContent, UrlType } from \"./types.ts\";\nimport { fetchTranscript } from \"npm:youtube-transcript\";\n\nconst JINA_API_KEY = Deno.env.get(\"JINA_API_KEY\");\nconst GITHUB_TOKEN = Deno.env.get(\"GITHUB_TOKEN\");\nconst FETCH_TIMEOUT = 10_000; // 10 seconds\n\n// ─── URL type detection ─────────────────────────────────────────────────────\nexport function detectUrlType(url: string): UrlType {\n const host = new URL(url).hostname.replace(/^www\\./, \"\").replace(/^m\\./, \"\");\n if (/^(twitter\\.com|x\\.com)$/.test(host)) return \"tweet\";\n if (/^(youtube\\.com|youtu\\.be)$/.test(host)) return \"youtube\";\n if (host === \"github.com\" && new URL(url).pathname.startsWith(\"/designatives/resources\")) return \"internal\";\n if (host === \"github.com\") return \"github\";\n if (host === \"arxiv.org\") return \"arxiv\";\n return \"article\";\n}\n\n// ─── Main extraction router ─────────────────────────────────────────────────\nexport async function extractContent(url: string): Promise {\n const type = detectUrlType(url);\n\n try {\n switch (type) {\n case \"tweet\":\n return { type, ...(await extractTweet(url)) };\n case \"youtube\":\n return { type, ...(await extractYouTube(url)) };\n case \"github\":\n return { type, ...(await extractGitHub(url)) };\n case \"internal\":\n return { type, ...(await extractInternalResource(url)) };\n default:\n return { type, ...(await extractArticle(url)) };\n }\n } catch (err) {\n console.error(`Extraction failed for ${url} (${type}):`, err);\n // Fallback: return minimal data so the pipeline can still attempt summarization\n return {\n type,\n title: new URL(url).hostname,\n content: `Failed to extract content from ${url}. Error: ${(err as Error).message}`,\n image: null,\n };\n }\n}\n\n// ─── Twitter/X extraction via FixTweet ──────────────────────────────────────\nasync function extractTweet(url: string): Promise> {\n const tweetIdMatch = url.match(/status\\/(\\d+)/);\n if (!tweetIdMatch) throw new Error(\"Could not extract tweet ID from URL\");\n\n const res = await fetchWithTimeout(\n `https://api.fxtwitter.com/i/status/${tweetIdMatch[1]}`,\n {},\n );\n const data = await res.json();\n const tweet = data.tweet;\n\n if (!tweet) throw new Error(\"FixTweet returned no tweet data\");\n\n // Prefer photo media, fall back to author avatar\n const image =\n tweet.media?.photos?.[0]?.url ||\n tweet.media?.videos?.[0]?.thumbnail_url ||\n tweet.author?.avatar_url ||\n null;\n\n return {\n title: `@${tweet.author.screen_name}: ${tweet.text.slice(0, 80)}...`,\n content: tweet.text + (tweet.quote ? `\\n\\n> Quoting @${tweet.quote.author.screen_name}: ${tweet.quote.text}` : \"\"),\n image,\n author: `${tweet.author.name} (@${tweet.author.screen_name})`,\n metrics: {\n likes: tweet.likes || 0,\n retweets: tweet.retweets || 0,\n replies: tweet.replies || 0,\n },\n };\n}\n\n// ─── YouTube extraction via oEmbed + transcript ────────────────────────────\nasync function extractYouTube(url: string): Promise> {\n const videoIdMatch = url.match(/(?:v=|youtu\\.be\\/|shorts\\/)([a-zA-Z0-9_-]{11})/);\n if (!videoIdMatch) throw new Error(\"Could not extract YouTube video ID from URL\");\n\n const videoId = videoIdMatch[1];\n const oembedUrl = `https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`;\n\n const res = await fetchWithTimeout(oembedUrl, {});\n const data = await res.json();\n\n // Attempt to fetch full transcript for richer AI summarization\n const transcript = await fetchYouTubeTranscript(videoId);\n\n const content = transcript\n ? `YouTube video by ${data.author_name}.\\nTitle: ${data.title}\\n\\nTranscript:\\n${transcript}`\n : `YouTube video by ${data.author_name}.\\n\\nTitle: ${data.title}`;\n\n return {\n title: data.title,\n content,\n image: `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,\n author: data.author_name,\n };\n}\n\nasync function fetchYouTubeTranscript(videoId: string): Promise {\n try {\n const segments = await fetchTranscript(videoId);\n if (!segments || segments.length === 0) return null;\n const fullText = segments.map((s: { text: string }) => s.text).join(\" \");\n return fullText.slice(0, 10_000);\n } catch {\n // Transcript unavailable (no captions, age-restricted, etc.)\n return null;\n }\n}\n\n// ─── GitHub repo extraction via REST API ────────────────────────────────────\nasync function extractGitHub(url: string): Promise> {\n const repoMatch = url.match(/github\\.com\\/([^\\/]+)\\/([^\\/\\?#]+)/);\n if (!repoMatch) throw new Error(\"Could not extract GitHub owner/repo from URL\");\n\n const [, owner, repo] = repoMatch;\n const repoName = repo.replace(/\\.git$/, \"\");\n\n const headers: Record = { \"User-Agent\": \"ai-feed/1.0\" };\n if (GITHUB_TOKEN) headers.Authorization = `token ${GITHUB_TOKEN}`;\n\n // Fetch repo metadata\n const repoRes = await fetchWithTimeout(\n `https://api.github.com/repos/${owner}/${repoName}`,\n { headers },\n );\n const repoData = await repoRes.json();\n\n // Try to fetch README content (first 2000 chars)\n let readmeContent = \"\";\n try {\n const readmeRes = await fetchWithTimeout(\n `https://raw.githubusercontent.com/${owner}/${repoName}/${repoData.default_branch || \"main\"}/README.md`,\n {},\n );\n if (readmeRes.ok) {\n const fullReadme = await readmeRes.text();\n readmeContent = fullReadme.slice(0, 2000);\n }\n } catch {\n // README fetch is optional\n }\n\n const topicsStr = (repoData.topics || []).join(\", \");\n\n return {\n title: repoData.full_name,\n content: [\n repoData.description || \"\",\n topicsStr ? `Topics: ${topicsStr}` : \"\",\n repoData.language ? `Language: ${repoData.language}` : \"\",\n readmeContent ? `\\n---\\nREADME excerpt:\\n${readmeContent}` : \"\",\n ]\n .filter(Boolean)\n .join(\"\\n\"),\n image: repoData.owner?.avatar_url || null,\n author: repoData.owner?.login || owner,\n metrics: {\n stars: repoData.stargazers_count || 0,\n forks: repoData.forks_count || 0,\n watchers: repoData.watchers_count || 0,\n },\n };\n}\n\n// ─── Internal resource extraction (designatives/resources) ─────────────────\nasync function extractInternalResource(url: string): Promise> {\n // Parse path: /designatives/resources/blob/main/path/to/file.ext\n const pathMatch = url.match(/\\/designatives\\/resources\\/(?:blob|tree)\\/[^\\/]+\\/(.+)/);\n const filePath = pathMatch?.[1] || \"\";\n const extension = filePath.split(\".\").pop() || \"\";\n\n const headers: Record = { \"User-Agent\": \"ai-feed/1.0\" };\n if (GITHUB_TOKEN) headers.Authorization = `token ${GITHUB_TOKEN}`;\n\n const contentsRes = await fetchWithTimeout(\n `https://api.github.com/repos/designatives/resources/contents/${filePath}`,\n { headers },\n );\n const contentsData = await contentsRes.json();\n\n // Fetch raw content for text files\n let fileContent = \"\";\n const textExtensions = [\"md\", \"txt\", \"json\", \"yaml\", \"yml\", \"ts\", \"js\", \"py\", \"html\", \"css\", \"csv\"];\n if (contentsData.download_url && textExtensions.includes(extension)) {\n try {\n const rawRes = await fetchWithTimeout(contentsData.download_url, {});\n fileContent = (await rawRes.text()).slice(0, 3_000);\n } catch {\n // Raw content fetch is optional\n }\n }\n\n return {\n title: `Internal: ${filePath || contentsData.name || \"resource\"}`,\n content: fileContent || `Internal resource at ${filePath}`,\n image: null,\n author: \"designatives\",\n description: `File type: .${extension}, Size: ${contentsData.size || 0} bytes`,\n };\n}\n\n// ─── Article extraction via Jina Reader + OG fallback ───────────────────────\nasync function extractArticle(url: string): Promise> {\n const jinaHeaders: Record = { Accept: \"application/json\" };\n if (JINA_API_KEY) jinaHeaders.Authorization = `Bearer ${JINA_API_KEY}`;\n\n const jinaRes = await fetchWithTimeout(`https://r.jina.ai/${url}`, {\n headers: jinaHeaders,\n });\n const jinaData = await jinaRes.json();\n\n // Extract OG image as fallback\n const ogImage = await fetchOGImage(url);\n\n return {\n title: jinaData.title || new URL(url).hostname,\n content: (jinaData.content || \"\").slice(0, 12_000),\n image: ogImage,\n author: jinaData.author || undefined,\n description: jinaData.description || undefined,\n };\n}\n\n// ─── OG image extraction ────────────────────────────────────────────────────\nasync function fetchOGImage(url: string): Promise {\n try {\n const res = await fetchWithTimeout(url, {\n headers: { \"User-Agent\": \"Mozilla/5.0 (compatible; ai-feed/1.0)\" },\n });\n const html = await res.text();\n\n // Match og:image in both attribute orderings\n const match =\n html.match(/property=[\"']og:image[\"'][^>]*content=[\"']([^\"']*)[\"']/i) ||\n html.match(/content=[\"']([^\"']*)[\"'][^>]*property=[\"']og:image[\"']/i);\n\n return match?.[1] || null;\n } catch {\n return null;\n }\n}\n\n// ─── Fetch with timeout helper ──────────────────────────────────────────────\nasync function fetchWithTimeout(\n url: string,\n init: RequestInit,\n timeout = FETCH_TIMEOUT,\n): Promise {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), timeout);\n\n try {\n const res = await fetch(url, { ...init, signal: controller.signal });\n if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);\n return res;\n } finally {\n clearTimeout(timer);\n }\n}\n","deleted","label",[176,177,159],"src","lib","permalink","https://www.val.town/x/Tamas_gr/ai-feed/code/src/lib/extract.ts","extension","ts","actionData","errors"]