📄 Policy Analysis Agents

\n Enter a document URL, paste text, or upload a PDF below. AI agents will analyze content, extract citations, and attempt reference traversal.
\n Uses OpenAI via Val Town & npm:pdf.js-extract for PDFs. Current Date: ${\n new Date().toLocaleDateString()\n }\n

\n\n

\n \n

\n\n

Analysis Results & Log

Processing...

\n\n

\n Powered by Val Town | View Source\n

\n\n \n\n`;\n}\n\n// --- Main Request Handler (Server Code) ---\nexport default async function(req: Request) {\n // --- Dynamic Imports (Inside Handler) ---\n const { OpenAI } = await import(\"https://esm.town/v/std/openai\");\n const { z } = await import(\"npm:zod\");\n const { fetch } = await import(\"https://esm.town/v/std/fetch\");\n // Import the PDF extraction library\n const { PDFExtract, PDFExtractOptions } = await import(\"npm:pdf.js-extract\");\n\n // --- Helper: Extract Text using pdf.js-extract ---\n /**\n * Extracts text from a PDF file buffer using npm:pdf.js-extract.\n * @param data The PDF file content as an ArrayBuffer.\n * @param fileName The name of the file for logging.\n * @param log Logger array to push status messages.\n * @returns Extracted text as a string, or null on failure.\n */\n async function extractPdfTextNative(\n data: ArrayBuffer,\n fileName: string,\n log: LogEntry[],\n ): Promise {\n const agent = \"PDF Extraction Agent (Native)\";\n log.push({ agent, type: \"step\", message: `Processing PDF: ${fileName} using npm:pdf.js-extract` });\n\n try {\n const pdfExtract = new PDFExtract();\n // Options can be configured if needed, see pdf.js-extract docs\n const options: PDFExtractOptions = {};\n\n // Perform extraction\n const pdfData = await pdfExtract.extractBuffer(data, options);\n\n // Combine text content from all pages\n // This joins text blocks on a page with spaces, and pages with double newlines\n const text = pdfData.pages.map(page => page.content.map(item => item.str).join(\" \")).join(\"\\n\\n\");\n\n if (!text || text.trim().length === 0) {\n log.push({ agent, type: \"error\", message: \"PDF extraction resulted in empty text content.\" });\n return null;\n }\n\n log.push({ agent, type: \"result\", message: `Successfully extracted ~${text.length} characters from PDF.` });\n return text;\n } catch (error) {\n const errorMessage = `Failed to extract text from PDF using pdf.js-extract: ${error.message}`;\n log.push({ agent, type: \"error\", message: errorMessage });\n console.error(agent, errorMessage, error); // Log the full error\n // Provide specific advice if the error suggests module issues\n if (error.message.includes(\"Cannot find module\") || error.message.includes(\"require is not defined\")) {\n log.push({\n agent: \"System\",\n type: \"error\",\n message:\n \"Error suggests 'npm:pdf.js-extract' might not be fully compatible with the Val Town runtime environment.\",\n });\n }\n return null;\n }\n }\n\n // --- Helper Function: Call OpenAI API (Unchanged) ---\n async function callOpenAI(\n openai: OpenAI,\n systemPrompt: string,\n userMessage: string,\n model = \"gpt-4o\",\n expectJson = false,\n ): Promise<{ role: \"assistant\" | \"system\"; content: string | object }> {\n try {\n const response = await openai.chat.completions.create({\n model: model,\n messages: [{ role: \"system\", content: systemPrompt }, { role: \"user\", content: userMessage }],\n temperature: 0.5,\n ...(expectJson && { response_format: { type: \"json_object\" } }),\n });\n const content = response.choices?.[0]?.message?.content;\n if (!content) throw new Error(\"Received empty content from AI model.\");\n if (expectJson) {\n try {\n return { role: \"assistant\", content: JSON.parse(content) };\n } catch (parseError) {\n console.error(\"OpenAI JSON Parse Error:\", parseError, \"Raw Content:\", content);\n throw new Error(`AI response was not valid JSON. Raw: ${content.substring(0, 150)}...`);\n }\n } else { return { role: \"assistant\", content: content }; }\n } catch (error) {\n console.error(`OpenAI API call failed. ExpectJSON: ${expectJson}. Error:`, error);\n let errorMessage = \"Error communicating with AI model.\";\n if (error.message) { errorMessage += ` Details: ${error.message}`; }\n return { role: \"system\", content: errorMessage };\n }\n }\n\n // --- Helper Function: Traverse Citations (Unchanged) ---\n async function traverseCitations(\n citations: Citation[],\n log: LogEntry[],\n ): Promise {\n const agent = \"Reference Traversal Agent\";\n log.push({ agent, type: \"step\", message: `Attempting to traverse ${citations.length} citations with URLs.` });\n const results: TraversedCitation[] = [];\n let traversedCount = 0;\n for (const citation of citations) {\n // Basic URL validation and prepending https:// if missing protocol\n let urlToFetch = citation.potentialUrl;\n if (urlToFetch) {\n try {\n // Attempt to construct a URL to validate/normalize it\n const parsedUrl = new URL(urlToFetch.startsWith(\"http\") ? urlToFetch : `https://${urlToFetch}`);\n urlToFetch = parsedUrl.href; // Use the normalized URL\n } catch (_) {\n log.push({ agent, type: \"error\", message: `Invalid potential URL format: ${citation.potentialUrl}` });\n results.push({ ...citation, traversalStatus: \"failed\", error: \"Invalid URL format\" });\n continue; // Skip to next citation\n }\n\n traversedCount++;\n let status: TraversedCitation[\"traversalStatus\"] = \"failed\";\n let errorMsg: string | undefined = undefined;\n try {\n log.push({ agent, type: \"step\", message: `Workspaceing reference: ${urlToFetch}` });\n const response = await fetch(urlToFetch, {\n headers: { \"User-Agent\": \"ValTownPolicyAnalysisBot/1.0\" },\n redirect: \"follow\",\n timeout: 10000,\n });\n if (!response.ok) throw new Error(`HTTP error! status: ${response.status} ${response.statusText}`);\n status = \"success\";\n log.push({ agent, type: \"result\", message: `Successfully accessed: ${urlToFetch}` });\n } catch (error) {\n errorMsg = `Failed to fetch reference ${urlToFetch}: ${error.message}`;\n log.push({ agent, type: \"error\", message: errorMsg });\n status = \"failed\";\n }\n results.push({ ...citation, potentialUrl: urlToFetch, traversalStatus: status, error: errorMsg }); // Store normalized URL\n } else { results.push({ ...citation, traversalStatus: \"url_missing\" }); }\n }\n if (traversedCount > 0)\n log.push({ agent, type: \"result\", message: `Finished traversal attempt for ${traversedCount} URLs.` });\n else log.push({ agent, type: \"step\", message: \"No valid citations with URLs were found to traverse.\" });\n\n return results;\n }\n\n // --- Main Agent Flow Logic (Using Native PDF Extraction) ---\n async function runAgentFlow(\n input: { documentUrl?: string; documentText?: string; documentFile?: File },\n log: LogEntry[],\n ): Promise {\n const openai = new OpenAI();\n\n log.push({ agent: \"System\", type: \"step\", message: \"Starting analysis workflow.\" });\n log.push({\n agent: \"System\",\n type: \"input\",\n message: `Input received (Type: ${input.documentFile ? \"File\" : (input.documentText ? \"Text\" : \"URL\")})`,\n });\n\n // --- 1. Ingestion (Using Native PDF Extraction) ---\n let documentText: string | null = null;\n const ingestionAgent = \"Ingestion Agent\";\n\n if (input.documentFile) {\n // Get ArrayBuffer from the File object\n try {\n const fileBuffer = await input.documentFile.arrayBuffer();\n // Call the native PDF extraction helper\n documentText = await extractPdfTextNative(fileBuffer, input.documentFile.name, log);\n // Error handling for extraction failure is now inside extractPdfTextNative\n if (!documentText) {\n log.push({ agent: \"System\", type: \"error\", message: \"Halting workflow: PDF extraction failed.\" });\n return log; // Return log containing the specific error from the helper\n }\n } catch (bufferError) {\n log.push({\n agent: ingestionAgent,\n type: \"error\",\n message: `Error reading PDF file buffer: ${bufferError.message}`,\n });\n documentText = null;\n }\n } else if (input.documentText) {\n log.push({ agent: ingestionAgent, type: \"result\", message: \"Using provided document text.\" });\n documentText = input.documentText;\n } else if (input.documentUrl) {\n log.push({ agent: ingestionAgent, type: \"step\", message: `Workspaceing from URL: ${input.documentUrl}` });\n try {\n const response = await fetch(input.documentUrl, { headers: { \"Accept\": \"text/plain, text/html\" } });\n if (!response.ok) throw new Error(`HTTP error! status: ${response.status} ${response.statusText}`);\n const text = await response.text();\n if (!text || text.trim().length === 0) throw new Error(\"Fetched content is empty or not text.\");\n log.push({\n agent: ingestionAgent,\n type: \"result\",\n message: `Workspaceed ~${text.length} characters from URL.`,\n });\n documentText = text;\n } catch (error) {\n const errorMessage = `Failed to fetch or process URL ${input.documentUrl}: ${error.message}`;\n log.push({ agent: ingestionAgent, type: \"error\", message: errorMessage });\n documentText = null;\n }\n }\n\n if (!documentText) {\n // An error should have already been logged during ingestion attempt\n if (!log.some(e => e.type === \"error\" && e.agent.includes(\"Ingestion\"))) {\n log.push({\n agent: \"System\",\n type: \"error\",\n message: \"Halting workflow: No document text available after ingestion attempts.\",\n });\n }\n return log;\n }\n\n // Limit text length\n const MAX_TEXT_LENGTH = 20000; // Consider limits for OpenAI context window\n const truncatedText = documentText.substring(0, MAX_TEXT_LENGTH);\n if (documentText.length > MAX_TEXT_LENGTH) {\n log.push({\n agent: \"System\",\n type: \"step\",\n message: `Document text truncated to ${MAX_TEXT_LENGTH} characters for analysis.`,\n });\n }\n\n // --- Steps 2, 3, 4, 5 (Analysis, Extraction, Traversal, Final Output) ---\n log.push({ agent: \"System\", type: \"step\", message: \"Starting Content Analysis...\" });\n const analysisResponse = await callOpenAI(openai, contentAnalysisSystemPrompt, truncatedText, \"gpt-4o\", true);\n let analysisResult: AnalysisResult | null = null;\n if (analysisResponse.role === \"assistant\" && typeof analysisResponse.content === \"object\") {\n // Basic validation of expected keys could be added here\n analysisResult = analysisResponse.content as AnalysisResult;\n log.push({ agent: \"Content Analysis Agent\", type: \"result\", message: analysisResult });\n } else { log.push({ agent: \"Content Analysis Agent\", type: \"error\", message: analysisResponse.content }); }\n\n log.push({ agent: \"System\", type: \"step\", message: \"Starting Citation Extraction...\" });\n const citationResponse = await callOpenAI(openai, citationExtractionSystemPrompt, truncatedText, \"gpt-4o\", true);\n let extractedCitations: Citation[] = [];\n if (\n citationResponse.role === \"assistant\" && typeof citationResponse.content === \"object\"\n && (citationResponse.content as any).citations\n ) {\n const citationData = citationResponse.content as CitationExtractionResult;\n if (Array.isArray(citationData.citations)) {\n extractedCitations = citationData.citations;\n log.push({\n agent: \"Citation Extraction Agent\",\n type: \"result\",\n message: { count: extractedCitations.length, citations: extractedCitations },\n });\n } else {log.push({\n agent: \"Citation Extraction Agent\",\n type: \"error\",\n message: \"AI response had 'citations' key, but it was not an array.\",\n });}\n } else if (citationResponse.role === \"assistant\") {\n log.push({\n agent: \"Citation Extraction Agent\",\n type: \"error\",\n message: `AI response was not the expected JSON format. Received: ${JSON.stringify(citationResponse.content)}`,\n });\n } else { log.push({ agent: \"Citation Extraction Agent\", type: \"error\", message: citationResponse.content }); }\n\n let traversedCitationsResult: TraversedCitation[] = [];\n if (extractedCitations.length > 0) {\n traversedCitationsResult = await traverseCitations(extractedCitations, log);\n } else {log.push({\n agent: \"Reference Traversal Agent\",\n type: \"step\",\n message: \"Skipping traversal - no citations.\",\n });}\n\n log.push({ agent: \"System\", type: \"final\", message: \"Workflow finished.\" });\n return log;\n }\n\n // --- Request Handling Logic ---\n const url = new URL(req.url);\n const format = url.searchParams.get(\"format\");\n const sourceUrl = import.meta.url.replace(\"esm.town\", \"val.town\");\n const CORS_HEADERS = {\n \"Access-Control-Allow-Origin\": \"*\",\n \"Access-Control-Allow-Methods\": \"POST, GET, OPTIONS\",\n \"Access-Control-Allow-Headers\": \"Content-Type, Accept\",\n };\n\n // --- CORS Preflight ---\n if (req.method === \"OPTIONS\") {\n return new Response(null, { status: 204, headers: CORS_HEADERS });\n }\n\n // --- API Request Branch (format=json and POST) ---\n if (format === \"json\" && req.method === \"POST\") {\n let inputData: { documentUrl?: string; documentText?: string; documentFile?: File } = {};\n const initialLog: LogEntry[] = [];\n\n try {\n const contentType = req.headers.get(\"content-type\");\n if (contentType && contentType.includes(\"multipart/form-data\")) {\n const formData = await req.formData();\n const file = formData.get(\"documentFile\");\n if (file instanceof File) {\n // Add more specific validation if needed (size, type already checked client-side but good practice here too)\n if (file.size === 0) throw new Error(\"Uploaded file is empty.\");\n if (!file.type || file.type !== \"application/pdf\") throw new Error(\"Uploaded file must be a PDF.\");\n // Consider adding server-side size limit too\n inputData.documentFile = file;\n // No need for separate log entry here, runAgentFlow logs input details\n } else { throw new Error(\"Expected a file ('documentFile') but none found in form data.\"); }\n } else if (contentType && contentType.includes(\"application/json\")) {\n const rawInput = await req.json();\n // Use Zod for robust JSON validation\n const JsonInputSchema = z.object({\n documentText: z.string().trim().min(1, { message: \"documentText cannot be empty\" }).optional(), // Ensure not just whitespace\n documentUrl: z.string().url({ message: \"Invalid URL format\" }).optional(),\n }).refine(data => data.documentText || data.documentUrl, {\n message: \"Either documentText or documentUrl must be provided in JSON.\",\n });\n inputData = JsonInputSchema.parse(rawInput);\n } else {throw new Error(\n `Unsupported Content-Type: ${contentType}. Must be application/json or multipart/form-data.`,\n );}\n } catch (error) {\n console.error(\"Input parsing/validation error:\", error);\n const details = (error instanceof z.ZodError) ? error.flatten().fieldErrors : error.message;\n initialLog.push({ agent: \"System\", type: \"error\", message: `Input Error: ${error.message}`, details: details });\n return new Response(JSON.stringify({ error: \"Invalid input.\", details: details, log: initialLog }), {\n status: 400,\n headers: { ...CORS_HEADERS, \"Content-Type\": \"application/json\" },\n });\n }\n\n try {\n const finalLog = await runAgentFlow(inputData, initialLog);\n // Determine status code based on whether critical errors occurred (e.g., ingestion failure)\n const hasCriticalError = finalLog.some(e => e.type === \"error\" && e.message.includes(\"Halting workflow\"));\n const statusCode = hasCriticalError ? 500 : 200;\n\n return new Response(JSON.stringify({ log: finalLog }), {\n status: statusCode,\n headers: { ...CORS_HEADERS, \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n // This catch block handles truly unexpected errors *within* runAgentFlow itself\n console.error(\"Critical error during runAgentFlow execution:\", error);\n initialLog.push({\n agent: \"System\",\n type: \"error\",\n message: `Unexpected Workflow Error: ${error.message}`,\n details: error.stack,\n });\n return new Response(\n JSON.stringify({ error: \"Agent processing failed unexpectedly.\", details: error.message, log: initialLog }),\n { status: 500, headers: { ...CORS_HEADERS, \"Content-Type\": \"application/json\" } },\n );\n }\n }\n // --- HTML Page Load Branch ---\n else {\n const initialUrlParam = url.searchParams.get(\"documentUrl\") || \"\";\n const initialTextParam = url.searchParams.get(\"documentText\") || \"\"; // Fixed typo here\n const html = generateHtmlShell(initialUrlParam, initialTextParam, sourceUrl);\n return new Response(html, { status: 200, headers: { \"Content-Type\": \"text/html\" } });\n }\n}","deleted","label",[99],"permalink","https://www.val.town/x/Get/Pathway/code/main.tsx","extension","actionData","errors"]