(re)Search tool

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_203":204,"_205":206},"routes/_app.x.$handle.$name",{"_95":94,"_32":44,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":57,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":25,"_48":49,"_50":49,"_51":-5,"_52":-5,"_18":53,"_54":55,"_56":57,"_58":57,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",32,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","chatStreaming.js","latestPath","fileId","330cd23c-73fb-11f0-adbc-0224a6c84d84","name","fileType","script","content","// Deno script and module: streaming chat completion test with SSE parsing and aggregation\n// - Reads GROQ_API_KEY from environment (when run as CLI)\n// - Exports streamChatAndAggregate(apiKey, payload, { logSse }) to allow server reuse\n// - CLI: logs raw SSE lines and prints aggregated JSON at the end\n\nimport \"jsr:@std/dotenv/load\";\n\nconst API_URL = \"https://api.groq.com/openai/v1/chat/completions\";\n\nfunction getApiKey() {\n const key = (typeof Deno !== \"undefined\") ? Deno.env.get(\"GROQ_API_KEY\") : null;\n if (!key) {\n console.error(\"Missing GROQ_API_KEY in environment.\");\n }\n return key;\n}\n\nfunction buildPayload(userQuery) {\n const prompt = typeof userQuery === \"string\" && userQuery.trim().length > 0\n ? userQuery\n : \"Stream test: please emit a short response and demonstrate reasoning and any tool usage.\";\n\n return {\n messages: [\n {\n role: \"user\",\n content: prompt,\n },\n ],\n model: \"openai/gpt-oss-120b\",\n stream: true,\n reasoning_effort: \"medium\",\n tools: [\n {\n type: \"browser_search\",\n },\n ],\n };\n}\n\n// Utility: Parse one Groq SSE JSON line (string without the leading \"data: \")\nfunction parseGroqSseJsonLine(jsonStr) {\n try {\n const obj = JSON.parse(jsonStr);\n return obj;\n } catch (_err) {\n return null;\n }\n}\n\n// Utility: Normalize a parsed chunk into events and collapsed accumulators\nfunction applyChunkToAggregates(parsed, aggregates) {\n const { events, collapsed } = aggregates;\n\n const timestamp = Date.now();\n const baseInfo = {\n id: parsed.id,\n model: parsed.model,\n created: parsed.created,\n instance_id: parsed.instance_id,\n system_fingerprint: parsed.system_fingerprint,\n timestamp,\n };\n\n if (Array.isArray(parsed.choices) && parsed.choices.length > 0) {\n const choice = parsed.choices[0];\n const delta = choice?.delta || {};\n\n if (delta.role) {\n events.push({ type: \"role\", role: delta.role, ...baseInfo });\n }\n\n if (delta.reasoning) {\n events.push({ type: \"reasoning\", content: delta.reasoning, channel: delta.channel, ...baseInfo });\n collapsed.reasoning += delta.reasoning;\n }\n\n if (delta.content) {\n events.push({ type: \"content\", content: delta.content, ...baseInfo });\n collapsed.content += delta.content;\n }\n\n if (delta.executed_tools) {\n events.push({ type: \"executed_tools\", executed_tools: delta.executed_tools, ...baseInfo });\n for (const t of delta.executed_tools) {\n collapsed.tools.push({ kind: \"executed\", ...t, timestamp });\n }\n }\n\n if (delta.tool_calls) {\n events.push({ type: \"tool_calls\", tool_calls: delta.tool_calls, ...baseInfo });\n for (const t of delta.tool_calls) {\n collapsed.tools.push({ kind: \"call\", ...t, timestamp });\n }\n }\n }\n}\n\nexport async function streamChatAndAggregate(apiKey, payload, options = {}) {\n const { logSse = false } = options;\n\n const response = await fetch(API_URL, {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${apiKey}`,\n },\n body: JSON.stringify(payload),\n });\n\n if (!response.ok) {\n const text = await response.text();\n throw new Error(`HTTP ${response.status} — ${text}`);\n }\n\n const reader = response.body?.getReader();\n const aggregates = {\n events: [],\n collapsed: { reasoning: \"\", content: \"\", tools: [] },\n };\n\n if (!reader) {\n try {\n const nonStream = await response.json();\n const content = nonStream?.choices?.[0]?.message?.content || \"\";\n if (content) {\n aggregates.events.push({ type: \"content\", content, timestamp: Date.now() });\n aggregates.collapsed.content = content;\n }\n } catch (_e) {\n // ignore\n }\n console.log(\"No readable stream; returning aggregates:\");\n console.log(JSON.stringify(aggregates, null, 2));\n return;\n }\n\n const decoder = new TextDecoder();\n let buffer = \"\";\n if (logSse) console.log(\"--- BEGIN SSE ---\");\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n buffer += decoder.decode(value, { stream: true });\n\n // Split Server-Sent Events by double newlines\n const events = buffer.split(\"\\n\\n\");\n buffer = events.pop() || \"\";\n\n for (const evt of events) {\n if (!evt.trim()) continue;\n // Extract `data:` line(s)\n const lines = evt.split(\"\\n\");\n for (const line of lines) {\n if (line.startsWith(\"data: \")) {\n const data = line.slice(6);\n if (data === \"[DONE]\") {\n if (logSse) console.log(\"[DONE]\");\n break;\n }\n if (logSse) {\n // Log raw SSE packet\n console.log(\"data:\", data);\n }\n // Parse and aggregate\n const parsed = parseGroqSseJsonLine(data);\n if (!parsed) continue;\n applyChunkToAggregates(parsed, aggregates);\n }\n }\n }\n }\n if (logSse) {\n console.log(\"--- END SSE ---\");\n console.log(\"Aggregates:\");\n console.log(JSON.stringify(aggregates, null, 2));\n }\n return aggregates;\n}\n\n// Entry point\nif (import.meta.main) {\n const apiKey = getApiKey();\n if (!apiKey) Deno.exit(1);\n\n const userQuery = Deno.args?.[0] || \"\";\n const payload = buildPayload(userQuery);\n\n streamChatAndAggregate(apiKey, payload, { logSse: true })\n .then(() => console.log(\"Stream complete.\"))\n .catch((err) => {\n console.error(\"Stream failed:\", err);\n Deno.exit(1);\n });\n}","createdAt",["D",1754682750954],"branchVersion","branchId","db71f2ae-724d-11f0-a35d-0224a6c84d84","id","db71c162-724d-11f0-a35d-0224a6c84d84","re-search",["D",1754433916817],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","ebafbf8e-724d-11f0-be81-0224a6c84d84","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",1,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":53,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"isOwned",42,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_42":43,"_32":44,"_37":199,"_46":-5,"_47":25,"_48":49,"_50":49,"_51":-5,"_52":-5,"_18":53,"_54":55,"_56":57,"_58":57,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":200,"_79":201,"_81":202,"_83":25},"tree",[178,179,180,181,182],"files",[178,179,180,181,182],"counts",{"_167":168,"_169":61,"_170":168,"_171":172,"_173":174},"contents",[125,126,127,128,129],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":160,"_32":161,"_30":162,"_132":-5,"_40":41,"_39":163,"_134":25,"_37":164,"_13":146,"_33":34,"_35":165,"_24":25,"_139":-5,"_27":166},{"_42":157,"_32":28,"_30":31,"_132":-5,"_40":41,"_39":14,"_134":25,"_37":158,"_13":146,"_33":34,"_35":36,"_24":25,"_139":-5,"_27":159},{"_42":149,"_32":150,"_30":151,"_132":-5,"_40":41,"_39":152,"_134":25,"_37":153,"_13":154,"_33":20,"_35":155,"_24":25,"_139":-5,"_27":156},{"_42":141,"_32":142,"_30":143,"_132":-5,"_40":41,"_39":144,"_134":25,"_37":145,"_13":146,"_33":34,"_35":147,"_24":25,"_139":-5,"_27":148},{"_42":130,"_32":131,"_30":55,"_132":-5,"_40":41,"_39":133,"_134":25,"_37":135,"_13":136,"_33":137,"_35":138,"_24":25,"_139":44,"_27":140},"296bd872-73fb-11f0-adbc-0224a6c84d84","main.js","parentId",18,"latest",["D",1754618301728],9,"http","import { Hono } from 'https://deno.land/x/hono@v3.11.12/mod.ts';\nimport { serve } from 'https://deno.land/std@0.190.0/http/server.ts';\nimport { listFiles, readFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\nimport { performSearch, summarizeUrl, buildSearchPrompt } from \"./chatCompletion.js\";\nimport { extractOrRepairJsonResults, extractResultsFromToolOutputs, sanitizeResults } from \"./jsonUtils.js\";\nimport { streamChatAndAggregate } from \"./chatStreaming.js\";\n\nconst app = new Hono();\nconst serverApiKey = null; // Deno.env.get(\"GROQ_API_KEY\");\n\n// Settings configuration\nconst settings = {\n model: 'openai/gpt-oss-120b',\n stream: false,\n reasoningEffort: 'low'\n};\n\n\n// Helper: Collect readable text from tool records for URL-grounded extraction\nfunction collectToolTextFromArray(toolArray) {\n if (!Array.isArray(toolArray)) return '';\n const parts = [];\n for (const t of toolArray) {\n try {\n const name = t?.name || t?.tool || t?.function?.name || t?.id || '';\n const args = t?.arguments || t?.input || t?.params || null;\n const output = t?.output || t?.result || t?.content || null;\n const searchResults = t?.search_results || t?.results || null;\n if (name) parts.push('name: ' + String(name));\n if (args) parts.push('arguments: ' + (typeof args === 'string' ? args : JSON.stringify(args)));\n if (output) parts.push('output: ' + (typeof output === 'string' ? output : JSON.stringify(output)));\n if (searchResults) parts.push('search_results: ' + (typeof searchResults === 'string' ? searchResults : JSON.stringify(searchResults)));\n } catch (_) {\n // ignore malformed tool objects\n }\n }\n return parts.join('\\n');\n}\n\n\n\n// Check if server has API key\napp.get('/api/check-key', (c) => {\n return c.json({ hasServerKey: !!serverApiKey });\n});\n\n// Environment-aware file reading function\nasync function readFileContent(filePath) {\n // Check if we're in Val Town environment\n if (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) {\n // Use Val Town's readFile\n return await readFile(filePath, import.meta.url);\n } else {\n // Use Deno's native readTextFile for local development\n return await Deno.readTextFile(filePath);\n }\n}\n\n// Serve root with HTML content\napp.get('/', async (c) => {\n try {\n const htmlContent = await readFileContent('index.html');\n return c.html(htmlContent);\n } catch (error) {\n console.error('Error reading HTML file:', error);\n return c.text('Error loading page', 500);\n }\n});\n\n\napp.post('/api/search', async (c) => {\n try {\n const body = await c.req.json();\n const { query, dateRange, source, language, userApiKey, offset = 0, url, title } = body || {};\n\n const apiKey = serverApiKey || userApiKey;\n if (!apiKey) {\n return c.json({ error: 'No Groq API key available. Please provide your own key.' }, 400);\n }\n\n // Unified handler: if URL present, run summary; otherwise require query and run search\n if (url && typeof url === 'string' && url.trim()) {\n const { summary, summaryTime } = await summarizeUrl({\n apiKey,\n url,\n title,\n language,\n model: settings.model,\n stream: settings.stream,\n reasoningEffort: settings.reasoningEffort,\n });\n return c.json({ summary, summaryTime, url, title, language });\n }\n\n if (!query || !query.trim()) {\n return c.json({ error: 'No search query provided' }, 400);\n }\n\n const { results, searchTime } = await performSearch({\n apiKey,\n query,\n dateRange,\n source,\n language,\n offset,\n model: settings.model,\n stream: settings.stream,\n reasoningEffort: settings.reasoningEffort,\n });\n\n\n console.log('>> results', results);\n return c.json({\n results: sanitizeResults(results),\n searchTime,\n query,\n dateRange,\n source,\n language,\n offset,\n });\n } catch (error) {\n console.error('Server error:', error);\n return c.json({ error: 'Server error: ' + error.message }, 500);\n }\n});\n\n// API: Raw SSE passthrough for chat completions stream (for @test_chat_stream.js)\napp.post('/api/chatStream', async (c) => {\n try {\n let body = {};\n try {\n body = await c.req.json();\n } catch (_) {\n body = {};\n }\n const {\n userApiKey,\n messages,\n query,\n dateRange,\n source,\n language,\n offset = 0,\n model = 'openai/gpt-oss-120b',\n reasoning_effort = 'medium',\n tools = [{ type: 'browser_search' }],\n temperature,\n top_p,\n stop,\n } = body || {};\n\n const apiKey = serverApiKey || userApiKey || Deno.env.get(\"GROQ_API_KEY\");\n if (!apiKey) {\n return c.json({ error: 'No Groq API key available. Please provide your own key.' }, 400);\n }\n\n let finalMessages = Array.isArray(messages) && messages.length > 0\n ? messages\n : (typeof query === 'string' && query.trim().length > 0\n ? [\n {\n role: 'system',\n content: 'Return ONLY JSON matching the schema; no markdown or commentary. For every result, url MUST be a direct deep link to the specific item/resource (not a homepage, category page, or search results page). Prefer canonical targets: external article for HN listings, PubMed PMID page or publisher DOI for papers, arXiv abs page, DOI resolver, exact thread permalinks for discussions, and precise GitHub paths. Never output root domains as url.'\n },\n { role: 'user', content: buildSearchPrompt(query, source, dateRange, language, offset) }\n ]\n : null);\n if (!finalMessages) {\n finalMessages = [{\n role: 'user',\n content: 'go to https://news.ycombinator.com/ and give me the first 5 stories'\n }];\n }\n\n const payload = {\n model,\n messages: finalMessages,\n stream: true,\n reasoning_effort,\n tools,\n ...(temperature !== undefined ? { temperature } : {}),\n ...(top_p !== undefined ? { top_p } : {}),\n ...(stop !== undefined ? { stop } : {}),\n };\n\n const aggregates = await streamChatAndAggregate(apiKey, payload, { logSse: false });\n let results = [];\n try {\n const textContent = aggregates?.collapsed?.content || '';\n const reasoningText = aggregates?.collapsed?.reasoning || '';\n const toolText = collectToolTextFromArray(aggregates?.collapsed?.tools || []);\n if (toolText && toolText.trim()) {\n // Prefer URLs grounded in actual tool outputs\n results = await extractResultsFromToolOutputs(toolText, apiKey);\n }\n if ((!results || results.length === 0) && textContent && textContent.trim()) {\n // Fallback to general JSON repair on model text\n results = await extractOrRepairJsonResults(textContent, apiKey);\n }\n if ((!results || results.length === 0) && reasoningText && reasoningText.trim()) {\n // Final fallback: use reasoning channel content\n results = await extractOrRepairJsonResults(reasoningText, apiKey);\n }\n } catch (_e) {\n // best-effort parsing; ignore errors\n }\n return c.json({ model, ...aggregates, results: sanitizeResults(results) });\n } catch (error) {\n console.error('chatStream error:', error);\n return c.json({ error: 'chatStream error: ' + error.message }, 500);\n }\n});\n\n// API: True SSE passthrough for chat completions stream (browser consumes incrementally)\napp.post('/api/chatStreamSSE', async (c) => {\n try {\n let body = {};\n try {\n body = await c.req.json();\n } catch (_) {\n body = {};\n }\n\n const {\n userApiKey,\n messages,\n query,\n dateRange,\n source,\n language,\n offset = 0,\n model = 'openai/gpt-oss-120b',\n reasoning_effort = 'medium',\n tools = [{ type: 'browser_search' }],\n temperature,\n top_p,\n stop,\n } = body || {};\n\n const apiKey = serverApiKey || userApiKey || Deno.env.get(\"GROQ_API_KEY\");\n if (!apiKey) {\n return c.json({ error: 'No Groq API key available. Please provide your own key.' }, 400);\n }\n\n let finalMessages = Array.isArray(messages) && messages.length > 0\n ? messages\n : (typeof query === 'string' && query.trim().length > 0\n ? [\n {\n role: 'system',\n content: 'Return ONLY JSON matching the schema; no markdown or commentary. For every result, url MUST be a direct deep link to the specific item/resource (not a homepage, category page, or search results page). Prefer canonical targets: external article for HN listings, PubMed PMID page or publisher DOI for papers, arXiv abs page, DOI resolver, exact thread permalinks for discussions, and precise GitHub paths. Never output root domains as url.'\n },\n { role: 'user', content: buildSearchPrompt(query, source, dateRange, language, offset) }\n ]\n : [{ role: 'user', content: 'go to https://news.ycombinator.com/ and give me the first 5 stories' }]);\n\n const payload = {\n model,\n messages: finalMessages,\n stream: true,\n reasoning_effort,\n tools,\n ...(temperature !== undefined ? { temperature } : {}),\n ...(top_p !== undefined ? { top_p } : {}),\n ...(stop !== undefined ? { stop } : {}),\n };\n\n const encoder = new TextEncoder();\n const stream = new ReadableStream({\n start: async (controller) => {\n try {\n const upstream = await fetch(\"https://api.groq.com/openai/v1/chat/completions\", {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': 'Bearer ' + apiKey,\n },\n body: JSON.stringify(payload),\n });\n\n if (!upstream.ok || !upstream.body) {\n let txt = '';\n try { txt = await upstream.text(); } catch (_) {}\n controller.enqueue(encoder.encode('event: error\\n'));\n controller.enqueue(encoder.encode('data: ' + JSON.stringify({ status: upstream.status, message: txt }) + '\\n\\n'));\n controller.enqueue(encoder.encode('data: [DONE]\\n\\n'));\n controller.close();\n return;\n }\n\n const reader = upstream.body.getReader();\n const decoder = new TextDecoder();\n let buffer = '';\n let collapsedContent = '';\n let collapsedReasoning = '';\n let collectedTools = [];\n\n // Initial comment to establish stream\n try { controller.enqueue(encoder.encode(': connected\\n\\n')); } catch (_) {}\n\n // Keep-alive pings\n const keepAliveId = setInterval(() => {\n try { controller.enqueue(encoder.encode(': keep-alive\\n\\n')); } catch (_) {}\n }, 15000);\n\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n buffer += decoder.decode(value, { stream: true });\n const events = buffer.split('\\n\\n');\n buffer = events.pop() || '';\n for (const evt of events) {\n if (!evt.trim()) continue;\n const lines = evt.split('\\n');\n let currentEvent = 'message';\n for (const line of lines) {\n if (line.startsWith('event: ')) {\n currentEvent = line.slice(7).trim();\n continue;\n }\n if (!line.startsWith('data: ')) continue;\n const data = line.slice(6);\n // Passthrough Groq SSE data lines\n controller.enqueue(encoder.encode('data: ' + data + '\\n\\n'));\n // Accumulate collapsed content to produce JSON results at end\n try {\n const parsed = JSON.parse(data);\n const choice = Array.isArray(parsed.choices) ? parsed.choices[0] : null;\n const delta = choice?.delta || {};\n if (delta.reasoning) collapsedReasoning += delta.reasoning;\n if (delta.content) collapsedContent += delta.content;\n if (Array.isArray(delta.executed_tools)) {\n collectedTools.push(...delta.executed_tools);\n }\n if (Array.isArray(delta.tool_calls)) {\n collectedTools.push(...delta.tool_calls);\n }\n } catch (_) {\n // ignore non-JSON payloads such as [DONE]\n }\n }\n }\n }\n // Signal end of stream – emit final structured results using LLM fixer\n try {\n let results = [];\n try {\n const toolText = collectToolTextFromArray(collectedTools);\n const textForExtraction = (collapsedContent && collapsedContent.trim())\n ? collapsedContent\n : ((collapsedReasoning && collapsedReasoning.trim()) ? collapsedReasoning : '');\n if (toolText && toolText.trim()) {\n results = await extractResultsFromToolOutputs(toolText, apiKey);\n }\n if ((!results || results.length === 0) && textForExtraction) {\n results = await extractOrRepairJsonResults(textForExtraction, apiKey);\n }\n } catch (_) {}\n controller.enqueue(encoder.encode('event: results\\n'));\n controller.enqueue(encoder.encode('data: ' + JSON.stringify({ results: sanitizeResults(results) }) + '\\n\\n'));\n } catch (_) {}\n controller.enqueue(encoder.encode('data: [DONE]\\n\\n'));\n } catch (err) {\n controller.enqueue(encoder.encode('event: error\\n'));\n controller.enqueue(encoder.encode('data: ' + JSON.stringify({ message: err?.message || String(err) }) + '\\n\\n'));\n } finally {\n clearInterval(keepAliveId);\n try { controller.close(); } catch (_) {}\n }\n } catch (outerErr) {\n try {\n controller.enqueue(encoder.encode('event: error\\n'));\n controller.enqueue(encoder.encode('data: ' + JSON.stringify({ message: outerErr?.message || String(outerErr) }) + '\\n\\n'));\n controller.enqueue(encoder.encode('data: [DONE]\\n\\n'));\n } catch (_) {}\n try { controller.close(); } catch (_) {}\n }\n }\n });\n\n return new Response(stream, {\n headers: {\n 'Content-Type': 'text/event-stream',\n 'Cache-Control': 'no-cache, no-transform',\n 'Connection': 'keep-alive',\n },\n });\n } catch (error) {\n console.error('chatStreamSSE error:', error);\n return c.json({ error: 'chatStreamSSE error: ' + error.message }, 500);\n }\n});\n\n// API endpoint for paper summarization\n// Removed legacy /api/summarize; summaries are handled by /api/search when a URL is provided\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;","label",[131],"33e05044-7491-11f0-9914-0224a6c84d84","jsonUtils.js","2bea9f98-73fb-11f0-82d7-0224a6c84d84",31,["D",1754682743779],3,"// General JSON extraction/repair helper using Groq Chat Completions\n// Converts arbitrary text into a strict JSON object and returns the `results` array\nexport async function extractOrRepairJsonResults(rawText, apiKey) {\n const response = await fetch('https://api.groq.com/openai/v1/chat/completions', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${apiKey}`,\n },\n body: JSON.stringify({\n model: 'openai/gpt-oss-120b',\n messages: [\n {\n role: 'system',\n content:\n 'You convert arbitrary text into a strict JSON object with a top-level `results` array. Return ONLY JSON, no markdown or comments. Output must already be sanitized: descriptions must be clean plain text with no citations (e.g., [1], (2)), no line numbers (e.g., L0:), no dagger symbols (†), no HTML, no code blocks, and no section headers. Descriptions must add new information beyond the title — never repeat or mirror the title. If a description would equal or start with the title, remove the title portion and ensure the remaining text adds value; if nothing remains, set description to null. Keep prose concise and readable.'\n },\n {\n role: 'user',\n content:\n 'Convert the following into ONLY valid JSON that matches the schema. If fields are missing, use null. Do not include markdown fences. In description, output plain text only (no citations, numbers-in-brackets, line numbers, or HTML).\\n---\\n' + rawText\n }\n ],\n response_format: {\n type: 'json_schema',\n json_schema: {\n name: 'search_results',\n schema: {\n type: 'object',\n properties: {\n results: {\n type: 'array',\n items: {\n type: 'object',\n properties: {\n title: { type: 'string' },\n url: { type: 'string' },\n description: { type: ['string', 'null'] },\n source: { type: ['string', 'null'] },\n published_at: { type: ['string', 'null'] },\n comments: { type: ['number', 'null'] },\n pmid: { type: ['string', 'null'] },\n doi: { type: ['string', 'null'] }\n },\n required: ['title', 'url'],\n additionalProperties: false\n }\n }\n },\n required: ['results'],\n additionalProperties: false\n }\n }\n },\n stream: false\n }),\n });\n if (!response.ok) {\n const text = await response.text();\n throw new Error(`Groq API error: ${response.status} ${text}`);\n }\n const data = await response.json();\n const jsonText = data.choices?.[0]?.message?.content || '{}';\n try {\n const parsed = JSON.parse(jsonText);\n let results = Array.isArray(parsed?.results) ? parsed.results : [];\n return results;\n } catch (_) {\n return [];\n }\n}\n\n// Extract results strictly from tool outputs (ground truth URLs only)\n// The tool logs may include lines like \"URL: ...\", markdown lists, tables, or raw text.\n// This helper instructs the model to ONLY use URLs that appear in the provided text and never invent new ones.\nexport async function extractResultsFromToolOutputs(toolText, apiKey) {\n const response = await fetch('https://api.groq.com/openai/v1/chat/completions', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${apiKey}`,\n },\n body: JSON.stringify({\n model: 'openai/gpt-oss-120b',\n messages: [\n {\n role: 'system',\n content:\n 'You will receive raw logs from browser tool executions that include titles and URLs. Critically: ONLY use URLs that literally appear in the logs. Do not fabricate or guess any URL. Produce a strict JSON object with a top-level `results` array. Each result should map a meaningful title to one of the URLs present in the logs. Prefer canonical deep links by rule: Hacker News → exact item permalink or the external article if explicitly listed; PubMed → PMID page; arXiv → /abs/; DOI → https://doi.org/...; Reddit/forums → exact thread permalink; GitHub → exact path to repo/issue/file/release. If a suitable title is not obvious, derive a concise one from nearby text. If description can be inferred, include a short clean sentence with no citations or artifacts; otherwise set description to null. Never include any URL not present in the logs.'\n },\n {\n role: 'user',\n content:\n 'From the following tool logs, extract results using ONLY URLs that occur in the text. Return ONLY valid JSON matching the schema. If fields are missing, use null. No markdown fences.\\n---\\n' + toolText\n }\n ],\n response_format: {\n type: 'json_schema',\n json_schema: {\n name: 'search_results_from_tools',\n schema: {\n type: 'object',\n properties: {\n results: {\n type: 'array',\n items: {\n type: 'object',\n properties: {\n title: { type: 'string' },\n url: { type: 'string' },\n description: { type: ['string', 'null'] },\n source: { type: ['string', 'null'] },\n published_at: { type: ['string', 'null'] },\n comments: { type: ['number', 'null'] },\n pmid: { type: ['string', 'null'] },\n doi: { type: ['string', 'null'] }\n },\n required: ['title', 'url'],\n additionalProperties: false\n }\n }\n },\n required: ['results'],\n additionalProperties: false\n }\n }\n },\n stream: false\n }),\n });\n if (!response.ok) {\n const text = await response.text();\n throw new Error(`Groq API error: ${response.status} ${text}`);\n }\n const data = await response.json();\n const jsonText = data.choices?.[0]?.message?.content || '{}';\n try {\n const parsed = JSON.parse(jsonText);\n const results = Array.isArray(parsed?.results) ? parsed.results : [];\n return results;\n } catch (_) {\n return [];\n }\n}\n\n// Remove null/undefined optional fields so UI never shows pmid: null or doi: null\nexport function sanitizeResults(results) {\n if (!Array.isArray(results)) return [];\n return results.map((r) => {\n const cleaned = { ...r };\n for (const key of ['pmid', 'doi', 'description', 'source', 'published_at', 'comments']) {\n if (cleaned[key] === null || cleaned[key] === undefined) {\n delete cleaned[key];\n }\n }\n return cleaned;\n });\n}\n\n\n\n// Structured summary extraction to eliminate citation artifacts and enforce clean fields\nexport async function extractStructuredSummary(rawText, apiKey) {\n const response = await fetch('https://api.groq.com/openai/v1/chat/completions', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${apiKey}`,\n },\n body: JSON.stringify({\n model: 'openai/gpt-oss-120b',\n messages: [\n {\n role: 'system',\n content:\n 'You convert arbitrary summary-like text into a strict JSON object describing the content. Return ONLY JSON, no markdown. Remove all citation artifacts (e.g., [1], (2), †), line numbers (e.g., L0:), and scraped UI fragments. Keep clean prose. Choose one type: research or general. For research, populate research_question, methods_and_findings, significance. For general, populate what_it_is, key_points (short bullet strings), and details. If any field would repeat or mirror the title or adds no value, set it to null. Keep text concise.'\n },\n {\n role: 'user',\n content:\n 'Normalize the following text into ONLY valid JSON matching the schema. No markdown fences, no HTML. Remove citations, numbers-in-brackets, line numbers, and artifacts.\\n---\\n' + rawText\n }\n ],\n response_format: {\n type: 'json_schema',\n json_schema: {\n name: 'structured_summary',\n schema: {\n type: 'object',\n properties: {\n type: { type: 'string', enum: ['general', 'research'] },\n general: {\n type: 'object',\n properties: {\n what_it_is: { type: 'string' },\n key_points: { type: 'array', items: { type: 'string' } },\n details: { type: 'string' }\n },\n additionalProperties: false\n },\n research: {\n type: 'object',\n properties: {\n research_question: { type: 'string' },\n methods_and_findings: { type: 'string' },\n significance: { type: 'string' }\n },\n additionalProperties: false\n }\n },\n required: ['type'],\n additionalProperties: false\n }\n }\n },\n stream: false\n }),\n });\n if (!response.ok) {\n const text = await response.text();\n throw new Error(`Groq API error: ${response.status} ${text}`);\n }\n const data = await response.json();\n const jsonText = data.choices?.[0]?.message?.content || '{}';\n try {\n const parsed = JSON.parse(jsonText);\n return parsed && typeof parsed === 'object' ? parsed : null;\n } catch (_) {\n return null;\n }\n}\n\nfunction escapeHtml(str) {\n if (!str) return '';\n return String(str)\n .replace(/&/g, '&')\n .replace(//g, '>')\n .replace(/\"/g, '"')\n .replace(/'/g, ''');\n}\n\n// Deterministic HTML renderer for structured summary\nexport function renderStructuredSummaryToHtml(structured) {\n if (!structured || typeof structured !== 'object') return '';\n const type = structured.type;\n if (type === 'research' && structured.research) {\n const rq = escapeHtml(structured.research.research_question || '');\n const mf = escapeHtml(structured.research.methods_and_findings || '');\n const sg = escapeHtml(structured.research.significance || '');\n let html = '';\n if (rq) html += '

Research Question: ' + rq + '

';\n if (mf) html += '

Methods & Findings: ' + mf + '

';\n if (sg) html += '

Significance: ' + sg + '

';\n return html;\n }\n // default to general\n const general = structured.general || {};\n const what = escapeHtml(general.what_it_is || '');\n const details = escapeHtml(general.details || '');\n const points = Array.isArray(general.key_points) ? general.key_points : [];\n let html = '';\n if (what) html += '

What it is: ' + what + '

';\n if (points.length > 0) {\n html += '

Key points:

' + escapeHtml(p) + '

';\n }\n if (details) html += '

Details: ' + details + '

';\n return html;\n}",[142],"6ac26f02-73fb-11f0-8705-0224a6c84d84","index.html","e0d474b0-724d-11f0-94bb-0224a6c84d84",29,["D",1754618411348],7,"\n\n\n \n \n (re)Search tool\n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n\n \n\n\n

\n\n

\n\n \n

\n \n \n

\n \n

\n\n \n

\n \n \n

\n \n

\n \n \n

\n \n \n \n

\n \n

\n \n \n

\n \n

\n \n \n

\n \n

\n \n \n

\n \n

\n\n \n

\n \n

\n \n \n

\n\n \n

0 || isStreaming\">\n

\n \n

\n\n \n

\n \n \n

0 && !isSearching\">\n

\n \n

\n \n \n

\n \n

\n\n\n

\n\n \n\n",[150],"3827c754-7491-11f0-8fd7-0224a6c84d84",["D",1754682750954],[28],"2db18da0-7491-11f0-9fdc-0224a6c84d84","chatCompletion.js","3d892ec2-73fb-11f0-9985-0224a6c84d84",30,["D",1754682733403],"// Minimal Groq Chat Completions client using fetch\nimport { extractOrRepairJsonResults, extractStructuredSummary, renderStructuredSummaryToHtml } from \"./jsonUtils.js\";\nexport async function groqChatCompletion(apiKey, payload) {\n console.log('>>> [groqChatCompletion] Payload:', payload);\n const response = await fetch('https://api.groq.com/openai/v1/chat/completions', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${apiKey}`,\n },\n body: JSON.stringify(payload),\n });\n if (!response.ok) {\n const text = await response.text();\n throw new Error(`Groq API error: ${response.status} ${text}`);\n }\n console.log('>>> [groqChatCompletion] Response:', response);\n return await response.json();\n}\n\n// Use LLM to normalize any freeform output into strict JSON that matches our schema.\n// extractOrRepairJsonResults moved to ./jsonUtils.js\n\nexport function buildSearchPrompt(query, source, dateRange, language, offset) {\n let languageInstruction = '';\n if (language && language !== 'english') {\n languageInstruction = `IMPORTANT: Follow this workflow: 1) First translate the search query \"${query}\" to English terms for better search accuracy. 2) Perform the search using only the English translation. 3) After finding results, translate all titles, descriptions, and content to ${language}. Always search in English but present results in ${language}.`;\n }\n\n const base = source && source.trim()\n ? `Search ${source} for: \"${query}\"`\n : `\"${query}\"`;\n\n let dateRangeText = '';\n if (dateRange) {\n dateRangeText = ({\n '1month': 'in the last month',\n '3months': 'in the last 3 months',\n '6months': 'in the last 6 months',\n '1year': 'in the last year',\n })[dateRange] || '';\n }\n\n const startResult = offset + 1;\n const endResult = offset + 5;\n const offsetInstruction = offset > 0 ? ` Skip the first ${offset} results and` : '';\n\n const searchPrompt = `${base}${dateRangeText ? ' ' + dateRangeText : ''}.${offsetInstruction} Return results ${startResult}-${endResult} (max 5 new results). Return ONLY JSON with a top-level \\\\\"results\\\\\" array. Each result must include: title and url. Optionally include: description (one concise sentence only if confidently grounded by the target page). Do NOT add or guess fields like source, published_at, comments, pmid, or doi — omit anything unknown.\\n\\nURL RULES (critical):\\n- Always provide a direct deep link to the specific item/resource — never a homepage, category page, or search results page.\\n- If the listing points to another resource, set url to the canonical content page, not the aggregator:\\n - Hacker News: prefer the external article URL if present; otherwise use the exact item permalink (https://news.ycombinator.com/item?id=...). Do not invent dates or comment counts.\\n - PubMed: use the PMID page (https://pubmed.ncbi.nlm.nih.gov/PMID/) or the publisher DOI landing page if available. Do not link to the PubMed homepage.\\n - arXiv: use https://arxiv.org/abs/ (or the PDF if explicitly asked for PDFs).\\n - DOI: use the https://doi.org/... resolver link.\\n - Reddit/Forums: use the exact thread permalink if the discussion is the result.\\n - GitHub: link to the exact repository, release, issue, or file path, not github.com root.\\n- Never return root domains (e.g., https://pubmed.ncbi.nlm.nih.gov/, https://news.ycombinator.com/). Ensure url resolves to the specific item. ${languageInstruction}`;\n return searchPrompt;\n}\n\n// Lightweight router: decide between 'links' (JSON list) and 'text' (short answer)\nexport async function routeQueryMode(apiKey, query) {\n try {\n const res = await groqChatCompletion(apiKey, {\n model: 'openai/gpt-oss-120b',\n messages: [\n { role: 'system', content: 'Classify the user request as either links or text. Respond with a single token: links or text. Use links if the user appears to want a search results list of sources; use text if the user is asking for an explanation/summary/definition.' },\n { role: 'user', content: String(query || '') }\n ],\n stream: false,\n });\n const out = (res?.choices?.[0]?.message?.content || '').trim().toLowerCase();\n if (out.includes('links')) return 'links';\n if (out.includes('text')) return 'text';\n } catch (_) { /* fall back to heuristic */ }\n // Heuristic fallback\n const q = String(query || '').trim().toLowerCase();\n const chatStarts = ['explain', 'what is', 'what are', 'how does', 'how do', 'summarize', 'overview', 'define', 'in simple terms', 'briefly', 'why is', 'compare', 'contrast'];\n if (chatStarts.some((s) => q.startsWith(s))) return 'text';\n if (q.endsWith('?') && q.length <= 200) return 'text';\n return 'links';\n}\n\nexport async function performSearch({ apiKey, query, dateRange, source, language, offset = 0, model, stream, reasoningEffort }) {\n const prompt = buildSearchPrompt(query, source, dateRange, language, offset);\n const start = Date.now();\n const route = await routeQueryMode(apiKey, query);\n const response = await groqChatCompletion(apiKey, {\n model,\n messages: route === 'text'\n ? [\n {\n role: 'system',\n content: 'If the user is asking for an explanation or summary rather than a list of links, respond with a concise plain-text answer in the requested language (no JSON). If the request appears to be a search for sources, return ONLY JSON with a top-level results array as previously specified.'\n },\n { role: 'user', content: query }\n ]\n : [\n {\n role: 'system',\n content: 'Return ONLY JSON matching the schema; no markdown or commentary. For every result, url MUST be a direct deep link to the specific item/resource (not a homepage, category page, or search results page). Prefer canonical targets: external article for HN listings, PubMed PMID page or publisher DOI for papers, arXiv abs page, DOI resolver, exact thread permalinks for discussions, and precise GitHub paths. Never output root domains as url.'\n },\n { role: 'user', content: prompt },\n ],\n stream: false,\n reasoning_effort: reasoningEffort,\n tools: [ { type: 'browser_search' } ],\n });\n const end = Date.now();\n const searchTime = (end - start) / 1000;\n const contentText = response.choices?.[0]?.message?.content || '';\n\n console.log('>> contentText', contentText);\n let results = [];\n if (route === 'links') {\n try {\n // Try direct parse first (in case the model already returned valid JSON)\n const stripped = String(contentText).replace(/^```json\\s*/i, '').replace(/```\\s*$/i, '');\n const parsed = JSON.parse(stripped);\n if (Array.isArray(parsed?.results)) results = parsed.results;\n } catch (_) {\n // as a fallback, attempt repair pass\n results = await extractOrRepairJsonResults(contentText, apiKey);\n }\n } else {\n // text mode → do not attempt to coerce content into results\n results = [];\n }\n // Also return assistant content and full response for upstream logging / fallback display\n return { results, searchTime, content: contentText, full: response, route };\n}\n\nimport { marked } from \"npm:marked\";\n\nfunction buildSummaryPrompt(url, title, language) {\n const languageInstruction = language && language !== 'english'\n ? `Please provide the summary in ${language}. `\n : '';\n\n return `Visit this URL: ${url}\n \nTitle: \"${title}\"\n\nCRITICAL INSTRUCTIONS:\n- Read the page content and produce a clean, professional summary.\n- Ignore navigation, comments, ads, headers/footers, and numerical artifacts.\n- If this is a research paper, use the RESEARCH format below.\n- If this is a news/blog/listing/discussion page (e.g., HN, Reddit, news site), use the GENERAL format below.\n- DO NOT include any citations, reference numbers, or raw scraping artifacts.\n- Write in clear prose without formatting artifacts.\n- The opening line of the summary must not simply repeat the title; avoid mirroring the title.\n\nReturn only one of the two structures below in markdown (choose based on page type). ${languageInstruction}\n\nRESEARCH FORMAT:\n

\n**Research Question:** [brief description of what the study investigates]\n\n**Methods & Findings:** [1-2 paragraphs covering methodology and key results - NO CITATIONS]\n\n**Significance:** [why this research matters and potential applications - NO CITATIONS]\n

\n\nGENERAL FORMAT:\n

\n**What it is:** [brief description of the page]\n\n**Key points:** [bullet list or short paragraphs with the main takeaways]\n\n**Details:** [1–2 short paragraphs with notable specifics]\n

`;\n}\n\nexport async function summarizeUrl({ apiKey, url, title, language, model, stream, reasoningEffort }) {\n const summaryPrompt = buildSummaryPrompt(url, title, language);\n const start = Date.now();\n const response = await groqChatCompletion(apiKey, {\n model,\n messages: [\n {\n role: 'system',\n content: 'You are a concise web content summarization expert. CRITICAL REQUIREMENTS: 1) Return ONLY the summary content within

tags - no other text, explanations, or metadata. 2) Use clean, professional markdown formatting. 3) NEVER include citations, reference numbers, line numbers, or any numerical markers like [1†, L0:, L1:, etc. 4) Write in clear prose without any formatting artifacts from source documents. 5) Adapt structure to page type and, if it is a research paper, strictly use the sections: Research Question, Methods & Findings, Significance; otherwise use the GENERAL structure.'\n },\n {\n role: 'user',\n content: summaryPrompt\n }\n ],\n stream,\n reasoning_effort: reasoningEffort,\n tools: [ { type: 'browser_search' } ],\n });\n const end = Date.now();\n const summaryTime = (end - start) / 1000;\n const responseText = response.choices[0]?.message?.content || '';\n\n // Prefer structured, JSON-cleaned summary to avoid leaking citation junk\n let cleanedHtml = '';\n try {\n const inner = (responseText.match(/

([\\s\\S]*?)<\\/summary>/) || [null, responseText])[1];\n const structured = await extractStructuredSummary(inner, apiKey);\n if (structured) {\n cleanedHtml = renderStructuredSummaryToHtml(structured);\n }\n } catch (_) {\n // ignore and fallback\n }\n if (!cleanedHtml) {\n // Final fallback: render markdown, but still try to strip outer

tags\n const match = responseText.match(/

([\\s\\S]*?)<\\/summary>/);\n const raw = match ? match[1].trim() : responseText;\n cleanedHtml = marked(raw);\n }\n\n return { summary: cleanedHtml, summaryTime };\n}",[161],"pulls",0,"branches","forks","envVars",{"_175":168,"_177":168},"connections",{"_175":168,"_176":168},"count","unfilledCount","emptyCount",{"_42":160,"_32":161,"_30":162,"_132":-5,"_40":41,"_39":163,"_134":25,"_37":196,"_13":146,"_33":34,"_35":165,"_24":25,"_139":-5,"_184":197,"_27":198},{"_42":157,"_32":28,"_30":31,"_132":-5,"_40":41,"_39":14,"_134":25,"_37":193,"_13":146,"_33":34,"_35":36,"_24":25,"_139":-5,"_184":194,"_27":195},{"_42":149,"_32":150,"_30":151,"_132":-5,"_40":41,"_39":152,"_134":25,"_37":190,"_13":154,"_33":20,"_35":155,"_24":25,"_139":-5,"_184":191,"_27":192},{"_42":141,"_32":142,"_30":143,"_132":-5,"_40":41,"_39":144,"_134":25,"_37":187,"_13":146,"_33":34,"_35":147,"_24":25,"_139":-5,"_184":188,"_27":189},{"_42":130,"_32":131,"_30":55,"_132":-5,"_40":41,"_39":133,"_134":25,"_37":183,"_13":136,"_33":137,"_35":138,"_24":25,"_139":44,"_184":185,"_27":186},["D",1754618301728],"children",[],[131],["D",1754682743779],[],[142],["D",1754618411348],[],[150],["D",1754682750954],[],[28],["D",1754682733403],[],[161],["D",1754433916817],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":53,"_99":100,"_101":25},{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"ctx",{"_77":-5,"_122":236,"_237":238,"_239":240,"_241":25,"_242":25,"_243":25,"_244":57,"_245":-5,"_246":-7,"_247":-7,"_248":168,"_249":25,"_250":25,"_251":-5,"_252":25,"_253":25,"_254":25,"_255":-5},"ENV",{"_207":208,"_209":210,"_211":85,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221,"_222":223,"_224":225,"_226":227,"_228":229,"_230":231,"_232":233,"_234":235},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","5015b8df46402d8bba4f485ecc37aec202b0e06b","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v32