🛡️ SafeSense Guardian-AI

Analyze text, images or documents and return a safety verdict + explanation.

\n\n

Moderate Text

\n \n

\n\n

Moderate Image / Document

\n \n

\n\n

Result

\n {loading &&

Checking...

}\n {result ? (\n

Safe: {String(result.safe)}

Categories: {(result.categories || []).join(', ') || '—'}

Short explanation: {result.summary}

\n {result.details && (\n

Detailed analysis

{JSON.stringify(result.details, null, 2)}

\n )}\n

\n ) : (\n

No result yet.

\n )}\n

\n\n \n

\n );\n}\n\nfunction client() {\n createRoot(document.getElementById(\"root\")).render();\n}\nif (typeof document !== 'undefined') client();\n\nexport default async function server(request: Request): Promise {\n // Single POST endpoint /moderate handles text, image, document\n if (request.method === 'POST' && new URL(request.url).pathname === '/moderate') {\n try {\n const body = await request.json();\n const { OpenAI } = await import('https://esm.town/v/std/openai');\n const openai = new OpenAI();\n\n // return structure\n const responsePayload: any = { safe: true, categories: [], summary: '', details: {} };\n\n if (body.type === 'text') {\n // Use moderation endpoint if available, else fallback to chat completion with explicit instructions.\n try {\n // Try moderation API\n const mod = await openai.moderations.create({ model: 'omni-moderation-latest', input: body.payload });\n responsePayload.details.moderation = mod;\n const flagged = (mod.results && mod.results[0] && mod.results[0].flagged) || false;\n responsePayload.safe = !flagged;\n responsePayload.categories = Object.keys(mod.results[0].categories || {}).filter(k => mod.results[0].categories[k]);\n responsePayload.summary = flagged ? 'Content flagged by moderation model.' : 'No issues found by moderation model.';\n } catch (e) {\n // Fallback: ask chat model to classify\n const chat = await openai.chat.completions.create({\n model: 'gpt-4o',\n messages: [\n { role: 'user', content: `Classify the following text as SAFE or UNSAFE. If UNSAFE, explain why and list categories (hate, sexual, violence, self-harm, illegal, other).\\n\\nText:\\n${body.payload}` }\n ]\n });\n const textResp = chat.choices[0].message.content;\n responsePayload.details.fallback = textResp;\n // best-effort parse\n responsePayload.summary = String(textResp).split('\\n')[0];\n responsePayload.safe = /safe/i.test(String(textResp).split('\\n')[0]) && !/unsafe/i.test(String(textResp));\n }\n }\n\n else if (body.type === 'image') {\n // Expect a data URL: data:image/...;base64,XXXX\n // We'll send the image to a vision-capable chat request and also run moderation on a short alt-text request.\n const dataUrl: string = body.payload;\n const b64 = dataUrl.split(',')[1];\n\n // Describe the image first\n try {\n const chat = await openai.chat.completions.create({\n model: 'gpt-4o',\n messages: [\n {\n role: 'user',\n content: [\n { type: 'text', text: 'Describe this image in detail: objects, colors, people, actions, and context. Then give a brief safety assessment: SAFE or UNSAFE and why.' },\n { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${b64}` } }\n ]\n }\n ]\n });\n const desc = chat.choices[0].message.content || '';\n responsePayload.details.description = desc;\n // Run moderation on the description + short prompt\n const mod = await openai.moderations.create({ model: 'omni-moderation-latest', input: desc });\n responsePayload.details.moderation = mod;\n const flagged = (mod.results && mod.results[0] && mod.results[0].flagged) || false;\n responsePayload.safe = !flagged;\n responsePayload.categories = Object.keys(mod.results[0].categories || {}).filter(k => mod.results[0].categories[k]);\n responsePayload.summary = flagged ? 'Image content flagged by moderation model.' : 'Image appears safe according to moderation model.';\n } catch (err) {\n responsePayload.safe = false;\n responsePayload.summary = 'Image analysis failed';\n responsePayload.details.error = String(err);\n }\n }\n\n else if (body.type === 'document') {\n // We received base64 file; for PDFs we could attempt to extract text, but here we send base64 + metadata to the model and ask for safety.\n try {\n // Ask the chat model to review the document and return safety summary. Provide the filename and a note that the content is base64 (so it should request text if needed).\n const docPrompt = `You are a content safety assistant. A document named \"${body.filename}\" with mime \"${body.mime}\" is provided as base64. Attempt a high-level safety analysis. If you can detect sexual content, hate, violence, self-harm, illegal instructions, or PII exposure, mark UNSAFE and explain why. Otherwise mark SAFE. Respond as JSON with keys: verdict (SAFE/UNSAFE), categories (list), summary (one-line), notes (detailed).\\n\\nBase64 (first 4000 chars): ${String(body.payload).slice(0,4000)}`;\n\n const chat = await openai.chat.completions.create({ model: 'gpt-4o', messages: [{ role: 'user', content: docPrompt }] });\n const textResp = chat.choices[0].message.content || '';\n responsePayload.details.review = textResp;\n // try to parse JSON from model\n try {\n const parsed = JSON.parse(textResp);\n responsePayload.safe = parsed.verdict === 'SAFE';\n responsePayload.categories = parsed.categories || [];\n responsePayload.summary = parsed.summary || '';\n responsePayload.details.notes = parsed.notes || parsed;\n } catch (parseErr) {\n responsePayload.summary = String(textResp).split('\\n')[0];\n responsePayload.safe = !/unsafe/i.test(String(textResp));\n responsePayload.details.parse_error = String(parseErr);\n }\n } catch (err) {\n responsePayload.safe = false;\n responsePayload.summary = 'Document analysis failed';\n responsePayload.details.error = String(err);\n }\n }\n\n return new Response(JSON.stringify(responsePayload), { headers: { 'Content-Type': 'application/json' } });\n } catch (error) {\n console.error('Moderation server error:', error);\n return new Response(JSON.stringify({ error: 'Server error', details: String(error) }), { status: 500, headers: { 'Content-Type': 'application/json' } });\n }\n }\n\n // Serve the client HTML shell when GET\n return new Response(`\n \n \n SafeSense Guardian-AI Content Moderator\n \n \n \n \n

\n \n \n \n \n `, { headers: { 'Content-Type': 'text/html' } });\n}\n\nconst css = `\nbody { font-family: 'Inter', Arial, sans-serif; background: #fbfbfd; margin: 0; }\nbutton { margin: 8px 6px 0 0; padding: 8px 12px; border-radius: 8px; border: none; background: #1f6feb; color: white; cursor: pointer; }\nbutton:hover { opacity: 0.95 }\n`;","deleted","label",[99],"permalink","https://www.val.town/x/mprasadhv/SafeSense_Guardian/code/main.ts","extension","actionData","errors"]