Input Document

\n \n

\n\n

\n \n

\n\n

\n \n \n \n \n \n \n \n \n \n \n \n

\n\n \n\n \n\n`;\n}\n\n// --- Main Request Handler (Server Code - Mostly Unchanged Backend Logic) ---\nexport default async function(req: Request) {\n // --- Dynamic Imports (Inside Handler) ---\n const { OpenAI } = await import(\"https://esm.town/v/std/openai\");\n const { z } = await import(\"npm:zod\");\n const { fetch } = await import(\"https://esm.town/v/std/fetch\");\n // Import the PDF extraction library\n const { PDFExtract, PDFExtractOptions } = await import(\"npm:pdf.js-extract\");\n\n // --- Helper: Extract Text using pdf.js-extract (Unchanged) ---\n async function extractPdfTextNative(\n data: ArrayBuffer,\n fileName: string,\n log: LogEntry[],\n ): Promise {\n const agent = \"PDF Extraction Agent (Native)\";\n log.push({ agent, type: \"step\", message: `Processing PDF: ${fileName} using npm:pdf.js-extract` });\n\n try {\n const pdfExtract = new PDFExtract();\n const options: PDFExtractOptions = {}; // Default options\n\n const pdfData = await pdfExtract.extractBuffer(data, options);\n\n // Combine text content from all pages\n const text = pdfData.pages.map(page => page.content.map(item => item.str).join(\" \") // Join text items on a page with spaces\n ).join(\"\\n\\n\"); // Join pages with double newlines\n\n if (!text || text.trim().length === 0) {\n log.push({ agent, type: \"error\", message: \"PDF extraction resulted in empty text content.\" });\n return null;\n }\n\n log.push({ agent, type: \"result\", message: `Successfully extracted ~${text.length} characters from PDF.` });\n return text;\n } catch (error) {\n const errorMessage = `Failed to extract text from PDF using pdf.js-extract: ${error.message}`;\n log.push({ agent, type: \"error\", message: errorMessage });\n console.error(agent, errorMessage, error);\n return null;\n }\n }\n\n // --- Helper Function: Call OpenAI API (Unchanged) ---\n async function callOpenAI(\n openai: OpenAI,\n systemPrompt: string,\n userMessage: string,\n model = \"gpt-4o\", // Use gpt-4o as default\n expectJson = false,\n ): Promise<{ role: \"assistant\" | \"system\"; content: string | object }> {\n try {\n const response = await openai.chat.completions.create({\n model: model,\n messages: [{ role: \"system\", content: systemPrompt }, { role: \"user\", content: userMessage }],\n temperature: 0.5, // Slightly creative but mostly factual\n ...(expectJson && { response_format: { type: \"json_object\" } }), // Request JSON format if needed\n });\n const content = response.choices?.[0]?.message?.content;\n if (!content) throw new Error(\"Received empty content from AI model.\");\n if (expectJson) {\n try {\n return { role: \"assistant\", content: JSON.parse(content) };\n } catch (parseError) {\n console.error(\"OpenAI JSON Parse Error:\", parseError, \"Raw Content:\", content);\n throw new Error(`AI response was not valid JSON. Raw: ${content.substring(0, 150)}...`);\n }\n } else { return { role: \"assistant\", content: content }; }\n } catch (error) {\n console.error(`OpenAI API call failed. ExpectJSON: ${expectJson}. Error:`, error);\n let errorMessage = \"Error communicating with AI model.\";\n if (error.message) { errorMessage += ` Details: ${error.message}`; }\n // Check for specific error types if needed (e.g., rate limits, auth)\n if (error.status === 401) errorMessage = \"Authentication error with OpenAI API. Check your secret key.\";\n if (error.status === 429) errorMessage = \"OpenAI API rate limit exceeded. Please try again later.\";\n\n return { role: \"system\", content: errorMessage };\n }\n }\n\n // --- Helper Function: Traverse Citations (Unchanged, logs results) ---\n async function traverseCitations(\n citations: Citation[],\n log: LogEntry[],\n ): Promise {\n const agent = \"Reference Traversal Agent\";\n log.push({ agent, type: \"step\", message: `Attempting to traverse ${citations.length} citations with URLs.` });\n const results: TraversedCitation[] = [];\n let traversedCount = 0;\n\n // Use Promise.allSettled for concurrency (optional, but can speed up)\n const traversalPromises = citations.map(async (citation): Promise => {\n if (citation.potentialUrl) {\n traversedCount++; // Increment here is slightly inaccurate if fetch fails early, but simpler\n let status: TraversedCitation[\"traversalStatus\"] = \"failed\";\n let errorMsg: string | undefined = undefined;\n let urlToFetch = citation.potentialUrl;\n // Basic URL validation/fixing\n if (!urlToFetch.startsWith('http://') && !urlToFetch.startsWith('https://')) {\n urlToFetch = 'http://' + urlToFetch; // Default to http if protocol missing\n }\n\n try {\n // Log the attempt for this specific URL\n log.push({ agent, type: \"step\", message: `Attempting to fetch: ${urlToFetch}` });\n const response = await fetch(urlToFetch, {\n headers: { \"User-Agent\": \"ValTownPolicyAnalysisBot/1.0 (+http://val.town)\" }, // Be a good bot citizen\n redirect: \"follow\", // Follow redirects\n timeout: 8000, // Shorter timeout (8 seconds)\n });\n if (!response.ok) throw new Error(`HTTP error! Status: ${response.status} ${response.statusText}`);\n // Optional: Check content-type if needed (e.g., ensure it's HTML/text)\n // const contentType = response.headers.get('content-type');\n // if (!contentType || (!contentType.includes('text/html') && !contentType.includes('text/plain'))) {\n // throw new Error(`Unexpected content type: ${contentType}`);\n // }\n status = \"success\";\n // Log success for this URL\n log.push({ agent, type: \"result\", message: `Successfully accessed: ${urlToFetch}` });\n return { ...citation, potentialUrl: urlToFetch, traversalStatus: status, error: undefined };\n } catch (error) {\n errorMsg = `Workspace failed for ${urlToFetch}: ${error.message.substring(0, 100)}`; // Keep error msg concise\n // Log failure for this URL\n log.push({ agent, type: \"error\", message: errorMsg });\n status = \"failed\";\n return { ...citation, potentialUrl: urlToFetch, traversalStatus: status, error: errorMsg };\n }\n } else {\n // No URL provided\n return { ...citation, traversalStatus: \"url_missing\" };\n }\n });\n\n // Wait for all attempts to settle (succeed or fail)\n const settledResults = await Promise.allSettled(traversalPromises);\n\n // Collect results\n settledResults.forEach(result => {\n if (result.status === 'fulfilled') {\n results.push(result.value);\n } else {\n // This case shouldn't happen often with try/catch inside, but log if it does\n console.error(\"Unexpected traversal promise rejection:\", result.reason);\n log.push({ agent, type: \"error\", message: `Internal error during citation traversal: ${result.reason}` });\n // Potentially push a placeholder error object if needed based on input citation\n }\n });\n\n\n if (traversedCount > 0)\n log.push({ agent, type: \"result\", message: `Finished traversal attempt for ${traversedCount} URLs. ${results.filter(r=>r.traversalStatus === 'success').length} succeeded.` });\n else log.push({ agent, type: \"step\", message: \"No citations with potential URLs were found to traverse.\" });\n\n // It's useful to log the final array of traversed citations for the frontend to parse\n log.push({ agent: agent, type: \"final\", message: results }); // Log the final array\n\n return results; // Return the array with statuses\n }\n\n\n // --- Main Agent Flow Logic (Unchanged - still logs steps) ---\n async function runAgentFlow(\n input: { documentUrl?: string; documentText?: string; documentFile?: File },\n log: LogEntry[],\n ): Promise { // Returns the completed log array\n const openai = new OpenAI(); // Assumes API key is in environment variables\n\n log.push({ agent: \"System\", type: \"step\", message: \"Starting analysis workflow.\" });\n log.push({\n agent: \"System\",\n type: \"input\",\n message: `Input received (Type: ${input.documentFile ? \"File\" : (input.documentText ? \"Text\" : \"URL\")})`,\n });\n\n // --- 1. Ingestion ---\n let documentText: string | null = null;\n const ingestionAgent = \"Ingestion Agent\";\n\n if (input.documentFile) {\n try {\n const fileBuffer = await input.documentFile.arrayBuffer();\n documentText = await extractPdfTextNative(fileBuffer, input.documentFile.name, log);\n if (!documentText) {\n // Error logged within extractPdfTextNative\n // Add a system log entry indicating halt\n log.push({ agent: \"System\", type: \"error\", message: \"Halting workflow: PDF extraction failed.\" });\n return log; // Return early\n }\n } catch (bufferError) {\n log.push({\n agent: ingestionAgent,\n type: \"error\",\n message: \\`Error reading PDF file buffer: \\${bufferError.message}\\`,\n });\n documentText = null; // Ensure it's null\n }\n } else if (input.documentText) {\n log.push({ agent: ingestionAgent, type: \"result\", message: \"Using provided document text.\" });\n documentText = input.documentText;\n } else if (input.documentUrl) {\n log.push({ agent: ingestionAgent, type: \"step\", message: \\`Workspaceing from URL: \\${input.documentUrl}\\` });\n try {\n // Basic check for common non-HTTP URL schemes that fetch will reject\n if (!input.documentUrl.match(/^https?:\\/\\//i)) {\n throw new Error(\"Invalid URL scheme. Only http and https are supported.\");\n }\n const response = await fetch(input.documentUrl, {\n headers: { \"Accept\": \"text/plain, text/html, application/pdf\", \"User-Agent\": \"ValTownPolicyAnalysisBot/1.0\" },\n redirect: \"follow\",\n timeout: 10000 // 10 second timeout for fetching\n });\n if (!response.ok) throw new Error(\\`HTTP error! Status: \\${response.status} \\${response.statusText}\\`);\n\n const contentType = response.headers.get(\"content-type\") || \"\";\n if (contentType.includes(\"application/pdf\")) {\n log.push({ agent: ingestionAgent, type: \"step\", message: \"Detected PDF content at URL. Attempting extraction.\"});\n const pdfBuffer = await response.arrayBuffer();\n documentText = await extractPdfTextNative(pdfBuffer, input.documentUrl, log);\n if (!documentText) {\n log.push({ agent: \"System\", type: \"error\", message: \"Halting workflow: PDF extraction from URL failed.\" });\n return log;\n }\n } else if (contentType.includes(\"text/html\") || contentType.includes(\"text/plain\") || contentType === \"\") { // Allow empty content type\n const text = await response.text();\n if (!text || text.trim().length === 0) throw new Error(\"Fetched content is empty or not text.\");\n log.push({\n agent: ingestionAgent,\n type: \"result\",\n message: \\`Workspaceed ~$\\{text.length} characters from URL.\\`,\n });\n documentText = text;\n // Optional: Add basic HTML stripping here if needed\n // documentText = documentText.replace(/<[^>]*>?/gm, ' '); // Basic strip\n } else {\n throw new Error(\\`Unsupported content type at URL: \\${contentType}\\`)\n }\n\n } catch (error) {\n const errorMessage = \\`Failed to fetch or process URL \\${input.documentUrl}: \\${error.message}\\`;\n log.push({ agent: ingestionAgent, type: \"error\", message: errorMessage });\n documentText = null;\n }\n }\n\n if (!documentText) {\n log.push({\n agent: \"System\",\n type: \"error\",\n message: \"Halting workflow: No document text available after ingestion.\",\n });\n return log; // Return early\n }\n\n // Limit text length for AI processing cost/speed\n const MAX_TEXT_LENGTH = 25000; // Increased limit slightly\n let truncatedText = documentText;\n if (documentText.length > MAX_TEXT_LENGTH) {\n truncatedText = documentText.substring(0, MAX_TEXT_LENGTH);\n log.push({\n agent: \"System\",\n type: \"step\",\n message: \\`Document text truncated to \\${MAX_TEXT_LENGTH} characters for analysis.\\`,\n });\n } else {\n log.push({ agent: \"System\", type: \"step\", message: \\`Using full document text (\\${documentText.length} characters) for analysis.\\` });\n }\n\n\n // --- 2. Content Analysis ---\n log.push({ agent: \"System\", type: \"step\", message: \"Starting Content Analysis...\" });\n const analysisResponse = await callOpenAI(openai, contentAnalysisSystemPrompt, truncatedText, \"gpt-4o\", true); // Use gpt-4o\n let analysisResult: AnalysisResult | null = null;\n if (analysisResponse.role === \"assistant\" && typeof analysisResponse.content === \"object\") {\n // Basic validation (can be enhanced with Zod)\n if (analysisResponse.content && typeof analysisResponse.content.summary === 'string' && Array.isArray(analysisResponse.content.mainThemes)) {\n analysisResult = analysisResponse.content as AnalysisResult;\n log.push({ agent: \"Content Analysis Agent\", type: \"result\", message: analysisResult });\n } else {\n log.push({ agent: \"Content Analysis Agent\", type: \"error\", message: \"AI response missing required fields or incorrect types.\" });\n console.error(\"Invalid analysis structure:\", analysisResponse.content);\n }\n } else {\n log.push({ agent: \"Content Analysis Agent\", type: \"error\", message: analysisResponse.content });\n // Optional: Halt workflow if analysis fails critically?\n // log.push({ agent: \"System\", type: \"error\", message: \"Halting workflow: Content Analysis failed.\" });\n // return log;\n }\n\n // --- 3. Citation Extraction ---\n log.push({ agent: \"System\", type: \"step\", message: \"Starting Citation Extraction...\" });\n const citationResponse = await callOpenAI(openai, citationExtractionSystemPrompt, truncatedText, \"gpt-4o\", true); // Use gpt-4o\n let extractedCitations: Citation[] = [];\n if (\n citationResponse.role === \"assistant\" && typeof citationResponse.content === \"object\"\n && (citationResponse.content as any).citations // Check if citations key exists\n ) {\n const citationData = citationResponse.content as CitationExtractionResult;\n if (Array.isArray(citationData.citations)) {\n // Basic validation of citation objects\n extractedCitations = citationData.citations.filter(c => c && typeof c.text === 'string');\n log.push({\n agent: \"Citation Extraction Agent\",\n type: \"result\",\n // Log the actual citations found for frontend parsing\n message: { count: extractedCitations.length, citations: extractedCitations },\n });\n } else {\n log.push({\n agent: \"Citation Extraction Agent\",\n type: \"error\",\n message: \"AI response had 'citations' key, but it was not an array.\",\n });\n console.error(\"Invalid citation structure:\", citationData);\n }\n } else if (citationResponse.role === \"assistant\") { // Got response, but not expected JSON\n log.push({\n agent: \"Citation Extraction Agent\",\n type: \"error\",\n message: \\`AI response was not the expected JSON format. Received: \\${JSON.stringify(citationResponse.content).substring(0, 200)}...\\`,\n });\n } else { // System error from callOpenAI\n log.push({ agent: \"Citation Extraction Agent\", type: \"error\", message: citationResponse.content });\n }\n\n // --- 4. Reference Traversal ---\n let traversedCitationsResult: TraversedCitation[] = [];\n if (extractedCitations.length > 0) {\n // Pass extracted citations to traversal function, it will log steps and return final array\n traversedCitationsResult = await traverseCitations(extractedCitations, log);\n // Traversal function already logs its final result array\n } else {\n log.push({\n agent: \"Reference Traversal Agent\",\n type: \"step\",\n message: \"Skipping traversal - no citations extracted.\",\n });\n // Ensure an empty array is logged if no citations were extracted initially\n log.push({ agent: \"Reference Traversal Agent\", type: \"final\", message: [] });\n }\n\n // --- 5. Final Output ---\n log.push({ agent: \"System\", type: \"final\", message: \"Workflow finished.\" });\n\n // Return the complete log. The frontend will parse this log.\n // TODO (Improvement): Ideally, construct a final result object here instead of just returning the log.\n // Example: return { analysis: analysisResult, citations: traversedCitationsResult, log: log };\n return log;\n }\n\n\n // --- Request Handling Logic ---\n const url = new URL(req.url);\n const format = url.searchParams.get(\"format\");\n const sourceUrl = import.meta.url.replace(\"esm.town\", \"val.town\"); // Get the Val Town URL\n const CORS_HEADERS = {\n \"Access-Control-Allow-Origin\": \"*\", // Allow requests from anywhere\n \"Access-Control-Allow-Methods\": \"POST, GET, OPTIONS\",\n \"Access-Control-Allow-Headers\": \"Content-Type, Accept\",\n };\n\n // --- CORS Preflight Handling ---\n if (req.method === \"OPTIONS\") {\n return new Response(null, { status: 204, headers: CORS_HEADERS });\n }\n\n // --- API Request Branch (POST with format=json) ---\n if (format === \"json\" && req.method === \"POST\") {\n let inputData: { documentUrl?: string; documentText?: string; documentFile?: File } = {};\n const initialLog: LogEntry[] = [];\n let statusCode = 200; // Default OK status\n\n try {\n const contentType = req.headers.get(\"content-type\");\n\n // --- Input Parsing (Form Data or JSON) ---\n if (contentType && contentType.includes(\"multipart/form-data\")) {\n const formData = await req.formData();\n const file = formData.get(\"documentFile\");\n const urlFromForm = formData.get(\"documentUrl\"); // Also check for URL/Text in form data\n const textFromForm = formData.get(\"documentText\");\n\n if (file instanceof File && file.size > 0) {\n if (file.type !== \"application/pdf\") throw new Error(\"Uploaded file must be a PDF.\");\n // Add size check here if needed, though frontend also checks\n inputData.documentFile = file;\n initialLog.push({ agent: \"System\", type: \"input\", message: \\`Received file: \\${file.name}\\` });\n } else if (typeof textFromForm === 'string' && textFromForm.trim()) {\n inputData.documentText = textFromForm.trim();\n initialLog.push({ agent: \"System\", type: \"input\", message: \"Received text input from form.\" });\n } else if (typeof urlFromForm === 'string' && urlFromForm.trim()) {\n // Basic URL validation\n try { new URL(urlFromForm); } catch (_) { throw new Error(\"Invalid Document URL provided in form.\"); }\n inputData.documentUrl = urlFromForm.trim();\n initialLog.push({ agent: \"System\", type: \"input\", message: \"Received URL input from form.\" });\n }\n else {\n // If file field exists but has no size, it might be an empty upload attempt\n if (file instanceof File && file.size === 0) {\n throw new Error(\"Uploaded file is empty.\");\n } else {\n throw new Error(\"No valid input (File, Text, or URL) found in form data.\");\n }\n }\n\n } else if (contentType && contentType.includes(\"application/json\")) {\n // Handle JSON input (less common for file uploads, but could be used for text/URL)\n const rawInput = await req.json();\n const JsonInputSchema = z.object({\n documentText: z.string().trim().min(1).optional(), // Ensure non-empty if provided\n documentUrl: z.string().trim().url(\"Invalid Document URL provided.\").optional(), // Validate URL format\n }).refine(data => data.documentText || data.documentUrl, {\n message: \"Either documentText or documentUrl must be provided in JSON.\",\n });\n // Use safeParse to handle validation errors gracefully\n const parsed = JsonInputSchema.safeParse(rawInput);\n if (!parsed.success) {\n // Combine Zod error messages\n const errorMessages = parsed.error.errors.map(e => e.message).join(', ');\n throw new Error(\\`Invalid JSON input: \\${errorMessages}\\`);\n }\n inputData = parsed.data;\n initialLog.push({\n agent: \"System\",\n type: \"input\",\n message: \\`Received JSON input (Type: \\${inputData.documentUrl ? \"URL\" : \"Text\"})\\`,\n });\n }\n // Allow application/x-www-form-urlencoded as well, similar to form-data text/url handling\n else if (contentType && contentType.includes(\"application/x-www-form-urlencoded\")){\n const formData = await req.formData(); // formData works for urlencoded too\n const urlFromForm = formData.get(\"documentUrl\");\n const textFromForm = formData.get(\"documentText\");\n if (typeof textFromForm === 'string' && textFromForm.trim()) {\n inputData.documentText = textFromForm.trim();\n initialLog.push({ agent: \"System\", type: \"input\", message: \"Received text input from urlencoded form.\" });\n } else if (typeof urlFromForm === 'string' && urlFromForm.trim()) {\n try { new URL(urlFromForm); } catch (_) { throw new Error(\"Invalid Document URL provided in urlencoded form.\"); }\n inputData.documentUrl = urlFromForm.trim();\n initialLog.push({ agent: \"System\", type: \"input\", message: \"Received URL input from urlencoded form.\" });\n } else {\n throw new Error(\"No valid input (Text or URL) found in urlencoded form data.\");\n }\n }\n else {\n throw new Error(\n \\`Unsupported Content-Type: \\${contentType}. Use multipart/form-data, application/json, or application/x-www-form-urlencoded.\\`,\n );\n }\n\n // --- Run Analysis ---\n const finalLog = await runAgentFlow(inputData, initialLog);\n\n // --- Determine Final Status Code ---\n const lastEntry = finalLog[finalLog.length - 1];\n // Consider it an internal error if the *System* ends with an error message indicating halt\n const systemHaltError = finalLog.some(entry => entry.agent === \"System\" && entry.type === \"error\" && entry.message?.toLowerCase().includes(\"halting workflow\"));\n if (systemHaltError) {\n statusCode = 500; // Internal Server Error\n } else {\n statusCode = 200; // OK\n }\n\n // --- Send Response ---\n // Return the full log, frontend will parse it\n return new Response(JSON.stringify({ log: finalLog }), {\n status: statusCode,\n headers: { ...CORS_HEADERS, \"Content-Type\": \"application/json\" },\n });\n\n } catch (error) {\n // Catch input parsing/validation errors or unexpected errors during flow start\n console.error(\"Input processing or critical workflow error:\", error);\n initialLog.push({ agent: \"System\", type: \"error\", message: \\`Input/Setup Error: \\${error.message}\\` });\n // Use 400 for client-side errors (bad input, bad content-type), 500 for others\n statusCode = (error.message.includes(\"Invalid\") || error.message.includes(\"Unsupported\") || error.message.includes(\"No valid input\")) ? 400 : 500;\n return new Response(JSON.stringify({ error: \"Failed to process request.\", details: error.message, log: initialLog }), {\n status: statusCode,\n headers: { ...CORS_HEADERS, \"Content-Type\": \"application/json\" },\n });\n }\n }\n // --- HTML Page Load Branch (GET Request or non-JSON POST) ---\n else {\n const initialUrlParam = url.searchParams.get(\"documentUrl\") || \"\";\n const initialTextParam = url.searchParams.get(\"documentText\") || \"\";\n // Generate the HTML shell with potentially pre-filled values and the Val's own URL\n const html = generateHtmlShell(initialUrlParam, initialTextParam, sourceUrl);\n return new Response(html, {\n status: 200,\n headers: { \"Content-Type\": \"text/html; charset=utf-8\" } // Specify UTF-8\n });\n }\n}\n","deleted","label",[98],"permalink","https://www.val.town/x/Get/Project/code/main.tsx","extension","actionData","errors"]