Autism Resource Tags Viewer

This page displays searchable tags extracted from autism resource documents. Tags are stored in an SQLite database (${DB_NAME}).

To add new documents for processing, send a POST request to ${sourceUrl}.\n See the POST endpoint documentation for payload format (e.g., via a GET request to the same URL with a non-browser User-Agent, or check the Val's code).

\n\n ${dbError ? `

Database Error: ${dbError}

` : \"\"}\n\n \n\n

All Extracted Tags

\n\n

Tags by Document

\n\n

\n \n\n \n\n`;\n}\n\n// --- SQLite Helper Functions (Refactored) ---\nasync function ensureTableExists(log: LogEntry[]) {\n const agent = \"SQLite Agent\";\n try {\n await sqlite.execute({\n sql: `CREATE TABLE IF NOT EXISTS autism_search_tags (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n document_identifier TEXT NOT NULL,\n tag TEXT NOT NULL,\n extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP,\n UNIQUE(document_identifier, tag)\n );`,\n args: [],\n });\n log.push({ agent, type: \"info\", message: \"Table 'autism_search_tags' ensured.\" });\n } catch (e) {\n log.push({ agent, type: \"error\", message: `Failed to ensure table: ${e.message}` });\n console.error(\"SQLite ensureTableExists error:\", e);\n throw e;\n }\n}\n\nasync function storeTags(documentIdentifier: string, tags: string[], log: LogEntry[]) {\n const agent = \"SQLite Agent\";\n if (!tags || tags.length === 0) {\n log.push({ agent, type: \"info\", message: \"No tags to store.\" });\n return 0;\n }\n let insertedCount = 0;\n log.push({\n agent,\n type: \"step\",\n message: `Attempting to store ${tags.length} tags for document: ${documentIdentifier}`,\n });\n\n for (const tag of tags) {\n if (!tag || typeof tag !== \"string\" || tag.trim() === \"\") {\n log.push({ agent, type: \"info\", message: `Skipping invalid or empty tag: '${tag}'` });\n continue;\n }\n try {\n const result = await sqlite.execute({\n sql: \"INSERT INTO autism_search_tags (document_identifier, tag) VALUES (?, ?)\",\n args: [documentIdentifier, tag.trim()],\n });\n // Check rowsAffected or lastInsertRowid; typically rowsAffected is for INSERT/UPDATE/DELETE\n // For INSERT, lastInsertRowid indicates success. rowsAffected might be 1 too.\n if (result && (result.rowsAffected > 0 || typeof result.lastInsertRowid === \"number\")) {\n insertedCount++;\n }\n } catch (e) {\n // More robust check for UNIQUE constraint violation if available (e.g., error code)\n if (\n e.message\n && (e.message.toLowerCase().includes(\"unique constraint failed\")\n || e.message.toLowerCase().includes(\"constraint failed: unique\"))\n ) {\n log.push({\n agent,\n type: \"info\",\n message: `Tag \"${tag}\" already exists for document \"${documentIdentifier}\". Skipping.`,\n });\n } else {\n log.push({ agent, type: \"error\", message: `SQLite insert error for tag \"${tag}\": ${e.message}` });\n console.error(`SQLite insert error for tag \"${tag}\":`, e);\n }\n }\n }\n log.push({ agent, type: \"result\", message: `Successfully stored ${insertedCount} new tags.` });\n return insertedCount;\n}\n\nexport default async function(req: Request) {\n // --- Dynamic Imports (Inside Handler for Val Town) ---\n const { OpenAI } = await import(\"https://esm.town/v/std/openai\");\n const val_fetch = (await import(\"https://esm.town/v/std/fetch\")).fetch;\n const { PDFExtract } = await import(\"npm:pdf.js-extract\");\n type PDFExtractOptions = import(\"npm:pdf.js-extract\").PDFExtractOptions;\n\n // ---Extract Text using pdf.js-extract (Unchanged from previous) ---\n async function extractPdfTextNative(data: ArrayBuffer, fileName: string, log: LogEntry[]): Promise {\n const agent = \"PDF Extraction Agent\";\n log.push({ agent, type: \"step\", message: `Processing PDF: ${fileName}...` });\n try {\n const pdfExtract = new PDFExtract();\n const options: PDFExtractOptions = {}; // Define options if needed, e.g., { firstPage: 1, lastPage: 5 }\n const pdfData = await pdfExtract.extractBuffer(data, options);\n const text = pdfData.pages.map(page => page.content.map(item => item.str).join(\" \")).join(\"\\n\\n\");\n if (!text?.trim()) {\n log.push({ agent, type: \"error\", message: \"PDF extraction resulted in empty text.\" });\n return null;\n }\n log.push({ agent, type: \"result\", message: `Extracted ~${text.length} chars from PDF.` });\n return text;\n } catch (error) {\n const errorMessage = `PDF extraction failed: ${error.message}`;\n log.push({ agent, type: \"error\", message: errorMessage });\n console.error(agent, errorMessage, error);\n return null;\n }\n }\n\n // --- Helper Function: Call OpenAI API (Unchanged from previous) ---\n async function callOpenAI(\n openai: OpenAI,\n systemPrompt: string,\n userMessage: string,\n model = \"gpt-4o\", // Updated to gpt-4o by default\n expectJson = false,\n ): Promise<{ role: \"assistant\" | \"system\"; content: string | object }> {\n try {\n const response = await openai.chat.completions.create({\n model,\n messages: [{ role: \"system\", content: systemPrompt }, { role: \"user\", content: userMessage }],\n temperature: 0.3, // Low temperature for more deterministic output\n ...(expectJson && { response_format: { type: \"json_object\" } }),\n });\n const content = response.choices?.[0]?.message?.content;\n if (!content) throw new Error(\"AI returned empty content.\");\n if (expectJson) {\n try {\n return { role: \"assistant\", content: JSON.parse(content) };\n } catch (e) {\n console.error(\"AI JSON Parse Error:\", e, \"Raw:\", content);\n throw new Error(`AI response not valid JSON: ${content.substring(0, 150)}`);\n }\n }\n return { role: \"assistant\", content };\n } catch (error) {\n console.error(`OpenAI call failed (ExpectJSON: ${expectJson}):`, error);\n let msg = \"Error communicating with AI.\";\n if (error.message) msg += ` Details: ${error.message}`;\n if ((error as any).status === 401) msg = \"OpenAI Auth Error. Check your 'openai' secret in Val Town.\";\n if ((error as any).status === 429) msg = \"OpenAI Rate Limit Exceeded.\";\n // Consider more specific error handling or re-throwing\n return { role: \"system\", content: msg }; // Or return an actual Error object / throw\n }\n }\n\n // --- Main Agent Flow Logic (Modified for SQLite refactor) ---\n async function runExtractionWorkflow(\n input: { documentUrl?: string; documentText?: string; documentFile?: File },\n log: LogEntry[],\n ): Promise<{ finalLog: LogEntry[]; extractedTagsCount: number; documentIdentifier: string | null }> {\n const openai = new OpenAI(); // Ensure your OpenAI secret is set in Val Town\n log.push({ agent: \"System\", type: \"step\", message: \"Workflow started.\" });\n\n let docText: string | null = null;\n let documentIdentifier: string | null = null;\n const ingestAgent = \"Ingestion Agent\";\n\n if (input.documentFile) {\n documentIdentifier = input.documentFile.name;\n log.push({ agent: \"System\", type: \"input\", message: `Input: File (${documentIdentifier})` });\n try {\n const buf = await input.documentFile.arrayBuffer();\n docText = await extractPdfTextNative(buf, documentIdentifier, log);\n } catch (e) {\n log.push({ agent: ingestAgent, type: \"error\", message: `PDF processing error: ${e.message}` });\n }\n } else if (input.documentText) {\n documentIdentifier = \"pasted_text_\" + Date.now();\n log.push({ agent: \"System\", type: \"input\", message: \"Input: Text\" });\n docText = input.documentText;\n log.push({ agent: ingestAgent, type: \"result\", message: `Using provided text (~${docText.length} chars).` });\n } else if (input.documentUrl) {\n documentIdentifier = input.documentUrl;\n log.push({ agent: \"System\", type: \"input\", message: `Input: URL (${documentIdentifier})` });\n log.push({ agent: ingestAgent, type: \"step\", message: `Workspaceing URL: ${input.documentUrl}` });\n try {\n if (!input.documentUrl.match(/^https?:\\/\\//i)) throw new Error(\"Invalid URL scheme. Must be http or https.\");\n const res = await val_fetch(input.documentUrl, {\n headers: {\n \"Accept\": \"text/plain, text/html, application/pdf\",\n \"User-Agent\": \"ValTownAutismTagExtractor/1.0\",\n },\n redirect: \"follow\",\n // timeout: 15000, // std/fetch might not support timeout directly like this\n });\n if (!res.ok) throw new Error(`HTTP error ${res.status} while fetching URL.`);\n const ct = res.headers.get(\"content-type\") || \"\";\n if (ct.includes(\"application/pdf\")) {\n log.push({ agent: ingestAgent, type: \"step\", message: \"URL content is PDF, extracting...\" });\n const buf = await res.arrayBuffer();\n docText = await extractPdfTextNative(buf, input.documentUrl, log);\n } else if (\n ct.includes(\"text/html\") || ct.includes(\"text/plain\") || ct === \"\" || ct.includes(\"application/json\")\n ) { // Added json for APIs\n docText = await res.text();\n if (!docText?.trim()) throw new Error(\"Fetched empty text from URL.\");\n if (ct.includes(\"text/html\")) {\n log.push({ agent: ingestAgent, type: \"step\", message: `HTML content detected. Attempting basic strip.` });\n // Basic HTML stripping, consider a more robust library for complex HTML\n docText = docText.replace(/]*>[\\s\\S]*?<\\/style>/gi, \"\");\n docText = docText.replace(/]*>[\\s\\S]*?<\\/script>/gi, \"\");\n docText = docText.replace(/<[^>]+>/g, \" \");\n docText = docText.replace(/\\s\\s+/g, \" \").trim();\n }\n log.push({ agent: ingestAgent, type: \"result\", message: `Workspaceed ~${docText.length} chars from URL.` });\n } else {\n throw new Error(`Unsupported content type from URL: ${ct}`);\n }\n } catch (e) {\n log.push({ agent: ingestAgent, type: \"error\", message: `URL fetch/process failed: ${e.message}` });\n }\n }\n\n if (!docText) {\n log.push({ agent: \"System\", type: \"error\", message: \"Halt: No document text could be obtained.\" });\n return { finalLog: log, extractedTagsCount: 0, documentIdentifier };\n }\n\n const MAX_CHARS_FOR_LLM = 30000; // Check model context window if using gpt-4o or other models\n let textToAnalyze = docText;\n if (docText.length > MAX_CHARS_FOR_LLM) {\n textToAnalyze = docText.substring(0, MAX_CHARS_FOR_LLM);\n log.push({\n agent: \"System\",\n type: \"step\",\n message: `Document text truncated from ${docText.length} to ${MAX_CHARS_FOR_LLM} characters for LLM analysis.`,\n });\n } else {\n log.push({\n agent: \"System\",\n type: \"step\",\n message: `Using full text (${docText.length} chars) for LLM analysis.`,\n });\n }\n\n log.push({ agent: \"System\", type: \"step\", message: \"Extracting searchable tags with LLM...\" });\n const tagAgent = \"Autism Tag Extraction Agent\";\n const llmResponse = await callOpenAI(openai, autismTagExtractionSystemPrompt, textToAnalyze, \"gpt-4o\", true);\n\n let extractedTags: string[] = [];\n if (llmResponse.role === \"assistant\" && llmResponse.content && typeof llmResponse.content === \"object\") {\n const result = llmResponse.content as TagExtractionResult;\n if (result.searchable_tags && Array.isArray(result.searchable_tags)) {\n extractedTags = result.searchable_tags.filter(tag => typeof tag === \"string\" && tag.trim() !== \"\");\n log.push({\n agent: tagAgent,\n type: \"result\",\n message: { count: extractedTags.length, tags_preview: extractedTags.slice(0, 10) },\n });\n } else {\n log.push({\n agent: tagAgent,\n type: \"error\",\n message: \"LLM response JSON does not contain 'searchable_tags' array or it's not an array.\",\n details: llmResponse.content,\n });\n }\n } else {\n log.push({\n agent: tagAgent,\n type: \"error\",\n message: `LLM call failed or returned unexpected format.`,\n details: llmResponse.content, // content here will be the error message from callOpenAI\n });\n }\n\n let storedCount = 0;\n if (extractedTags.length > 0 && documentIdentifier) {\n log.push({ agent: \"System\", type: \"step\", message: \"Storing extracted tags in SQLite...\" });\n try {\n await ensureTableExists(log); // No db object passed\n storedCount = await storeTags(documentIdentifier, extractedTags, log); // No db object passed\n } catch (e) {\n log.push({ agent: \"System\", type: \"error\", message: `Database operation failed: ${e.message}` });\n }\n } else if (!documentIdentifier) {\n log.push({ agent: \"System\", type: \"error\", message: \"Cannot store tags: Document identifier is missing.\" });\n } else if (extractedTags.length === 0) {\n log.push({ agent: \"System\", type: \"info\", message: \"No tags were extracted by the LLM to store.\" });\n }\n\n log.push({ agent: \"System\", type: \"final\", message: \"Workflow finished.\" });\n return { finalLog: log, extractedTagsCount: storedCount, documentIdentifier };\n }\n\n // --- Request Handling Logic ---\n const currentUrlPath = new URL(req.url).pathname; // Use for simple routing if needed\n const CORS_HEADERS = {\n \"Access-Control-Allow-Origin\": \"*\",\n \"Access-Control-Allow-Methods\": \"POST, GET, OPTIONS\",\n \"Access-Control-Allow-Headers\": \"Content-Type, Accept\",\n };\n\n if (req.method === \"OPTIONS\") {\n return new Response(null, { status: 204, headers: CORS_HEADERS });\n }\n\n // POST handler (for data ingestion)\n if (req.method === \"POST\") {\n let inputData: { documentUrl?: string; documentText?: string; documentFile?: File } = {};\n const initialLog: LogEntry[] = [];\n let statusCode = 200;\n let responseBody: object;\n\n try {\n const contentType = req.headers.get(\"content-type\");\n if (contentType?.includes(\"multipart/form-data\")) {\n const formData = await req.formData();\n const file = formData.get(\"documentFile\");\n const urlVal = formData.get(\"documentUrl\");\n const textVal = formData.get(\"documentText\");\n\n if (file instanceof File && file.size > 0) {\n if (file.type !== \"application/pdf\") {\n throw new Error(\"Invalid file type. Only PDF is supported for file uploads.\");\n }\n if (file.size > MAX_FILE_SIZE_MB * 1024 * 1024) {\n throw new Error(`File too large. Max size is ${MAX_FILE_SIZE_MB}MB.`);\n }\n inputData.documentFile = file;\n } else if (typeof textVal === \"string\" && textVal.trim()) {\n inputData.documentText = textVal.trim();\n } else if (typeof urlVal === \"string\" && urlVal.trim()) {\n try {\n new URL(urlVal.trim());\n } catch (_) {\n throw new Error(\"Invalid Document URL provided in form data.\");\n }\n inputData.documentUrl = urlVal.trim();\n } else {\n throw new Error(\"No valid input (documentFile, documentText, or documentUrl) found in form data.\");\n }\n } else if (contentType?.includes(\"application/json\")) {\n const jsonBody = await req.json();\n if (jsonBody.documentText && typeof jsonBody.documentText === \"string\" && jsonBody.documentText.trim()) {\n inputData.documentText = jsonBody.documentText.trim();\n } else if (jsonBody.documentUrl && typeof jsonBody.documentUrl === \"string\" && jsonBody.documentUrl.trim()) {\n try {\n new URL(jsonBody.documentUrl.trim());\n } catch (_) {\n throw new Error(\"Invalid Document URL provided in JSON.\");\n }\n inputData.documentUrl = jsonBody.documentUrl.trim();\n } else {\n throw new Error(\"JSON body must contain 'documentText' or 'documentUrl'.\");\n }\n } else {\n throw new Error(\n `Unsupported Content-Type: ${contentType}. Please use 'multipart/form-data' or 'application/json'.`,\n );\n }\n\n if (!inputData.documentFile && !inputData.documentText && !inputData.documentUrl) {\n // This case should ideally be caught by the specific checks above\n throw new Error(\"No document input provided.\");\n }\n\n const { finalLog, extractedTagsCount, documentIdentifier } = await runExtractionWorkflow(inputData, initialLog);\n\n const hasSystemHalt = finalLog.some(e =>\n e.agent === \"System\" && e.type === \"error\" && e.message?.includes(\"Halt\")\n );\n if (hasSystemHalt) statusCode = 500;\n else if (finalLog.some(e => e.type === \"error\")) statusCode = 500; // General error\n else statusCode = 200;\n\n responseBody = {\n message: extractedTagsCount > 0\n ? `Successfully processed document '${documentIdentifier}' and stored ${extractedTagsCount} new tags.`\n : `Processed document '${documentIdentifier}'. No new tags were stored (either none extracted or all were duplicates).`,\n documentIdentifier: documentIdentifier,\n tagsStored: extractedTagsCount,\n log: finalLog,\n };\n } catch (error) {\n console.error(\"Request handling error:\", error);\n initialLog.push({ agent: \"System\", type: \"error\", message: `Request Processing Error: ${error.message}` });\n statusCode = (error.message.includes(\"Invalid\") || error.message.includes(\"Unsupported\")\n || error.message.includes(\"No valid input\") || error.message.includes(\"must contain\"))\n ? 400\n : 500;\n responseBody = { error: \"Failed to process request.\", details: error.message, log: initialLog };\n }\n\n return new Response(JSON.stringify(responseBody), {\n status: statusCode,\n headers: { ...CORS_HEADERS, \"Content-Type\": \"application/json\" },\n });\n }\n\n // GET handler (for UI - Modified for SQLite refactor)\n if (req.method === \"GET\") {\n let tagsData: StoredTag[] = [];\n let dbError: string | undefined = undefined;\n try {\n // Ensure table exists (no log needed for UI path, errors will be caught)\n await sqlite.execute({\n sql:\n `CREATE TABLE IF NOT EXISTS autism_search_tags (id INTEGER PRIMARY KEY AUTOINCREMENT, document_identifier TEXT NOT NULL, tag TEXT NOT NULL, extracted_at DATETIME DEFAULT CURRENT_TIMESTAMP, UNIQUE(document_identifier, tag));`,\n args: [],\n });\n\n const result = await sqlite.execute({\n sql:\n \"SELECT id, document_identifier, tag, extracted_at FROM autism_search_tags ORDER BY extracted_at DESC, document_identifier, tag\",\n args: [],\n });\n\n tagsData = (result.rows as StoredTag[]) || []; // sqlite.execute usually returns { rows: [...] }\n } catch (e) {\n console.error(\"Error fetching tags for UI:\", e);\n dbError = e.message;\n tagsData = [];\n }\n\n if (!Array.isArray(tagsData)) { // Should be redundant if result.rows is always an array\n tagsData = [];\n }\n\n const html = generateViewerHtml(tagsData, dbError);\n return new Response(html, {\n status: 200,\n headers: { ...CORS_HEADERS, \"Content-Type\": \"text/html; charset=utf-8\" },\n });\n }\n\n // Fallback for other methods\n return new Response(\"Method Not Allowed\", { status: 405, headers: CORS_HEADERS });\n}","deleted","label",[99],"permalink","https://www.val.town/x/salon/untitled-4923/code/main.tsx","extension","actionData","errors"]