Search

export default async function semanticSearchPublicVals(query) {
  const client = new Client(Deno.env.get("NEON_URL_VALSEMBEDDINGS"));
  await client.connect();
  const openai = new OpenAI();
  const queryEmbedding = (await openai.embeddings.create({
    model: "text-embedding-3-small",
  const result = await client
    .queryObject`SELECT id, embedding <=> ${embeddedBinaryString} AS distance FROM vals_embeddings ORDER BY embedding <=> ${embeddedBinaryString} LIMIT 50`;

// CREATE TABLE vals_embeddings (id TEXT PRIMARY KEY, embedding VECTOR(1536));
  const client = new Client(Deno.env.get("NEON_URL_VALSEMBEDDINGS"));
  await client.connect();
  const existingEmbeddingsIds = new Set(
    (await client.queryObject`SELECT id FROM vals_embeddings`).rows.map(row => row.id),
  );
    const id = idForVal(val);
    if (!existingEmbeddingsIds.has(id)) {
      currentBatch.push(val);
      const embedding = await openai.embeddings.create({
        model: "text-embedding-3-small",
      const result = await client
        .queryObject`INSERT INTO vals_embeddings (id, embedding) VALUES (${id}, ${embeddedBinaryString})`;
      console.log(`Processed ${id}..`);

Migrated from folder: semanticSearchBlogPost/compareEmbeddings

export const getBlurbFromUrl = async (url: string, {
  getSummary=true, getSummaryPrompt, getTags=true, getTagsPrompt, getEmbeddings} = {}) => {
  const content = await getContentFromUrl(url);
  let summary, tags, embeddings
  if(getSummary) {
  }
  if(getEmbeddings) {
    embeddings = await getEmbeddingsFn(content);
  }
    tags: tags.text,
    embeddings
  }
export const getEmbeddingsFn = async (content) => {
  let result = await ai({
  const body = await c.req.json();
  const { url, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body;
  const content = await getContentFromUrl(url);
  let summary, tags, embeddings
  if(getSummary) {
  }
  if(getEmbeddings) {
    embeddings = await getEmbeddingsFn(content);
  }
  
  return c.json({ summary: summary.text, tags: tags.text, content, embeddings,  });
});

Todo
- Support lunr / semantic search, and embeddings
- Collections that support pointing to multiple blobs, like {description, embeddings, fileblob, ...} with a shared index / lookup

  async addDocuments({ documents, fields = 'content', modelName = 'text-embedding-3-large' }) {
    const documentsWithoutEmbeddings = documents?.filter(doc => !doc.embedding) || [];
    const documentsWithEmbeddings = documents?.filter(doc => doc.embedding) || [];
    if (documentsWithoutEmbeddings.length > 0) {
      const contents = documentsWithoutEmbeddings.map(doc => doc.content);
      const embeddings = await this.getEmbeddings(contents, modelName);
      documentsWithoutEmbeddings.forEach((doc, index) => {
        doc.embedding = embeddings[index];
        this.documents.push(doc);
    documentsWithEmbeddings.forEach(doc => {
      this.documents.push(doc);
  async getEmbeddings(texts, modelName) {
    const { embeddings } = await embedMany({
      model: openai.embedding(modelName),
    });
    return embeddings;
  }

          <th>Content</th>
          <th>embeddingsContent</th>
        </tr>
            <td>{result.content}</td>
            <td>{result.embeddingsContent}</td>
          </tr>
const defaultContentColumn = "Content";
const defaultEmbeddingColumn = "Embeddings";
    embedding: record.get(embeddingColumn).split(",").map(parseFloat),
    embeddingsContent: record.get('EmbeddingsContent'),
  }));
    console.log('documents:', documents)
    await semanticSearch.addDocuments({documents, fields: 'embeddingsContent'});
    const results = await semanticSearch.search({query, similarityThreshold});

Demo of adding an Embeddings column to Airtable (which calls the embeddings endpoint at yawnxyz/v/ai) and stores embeddings in an Airtable column, then performing search against it

const defaultContentColumn = "Content";
const defaultEmbeddingColumn = "Embeddings";

Use embeddings / Lunr search on Airtable. Embeddings need to have been generated / stored on Airtable, or this gets very slow / costly.
- Simple usage: https://yawnxyz-buildclubprojectsearch.web.val.run/search?query=cars

// Step 1: Get Embeddings
// Function to get a single embedding
// Function to get embeddings for multiple texts
async function getEmbeddings(texts) {
  console.log(`Getting embeddings for texts: ${texts}`);
  const { embeddings } = await embedMany({
    model: openai.embedding('text-embedding-3-small'),
  });
  console.log(`Embeddings: ${embeddings}`);
  return embeddings;
}
// Step 2: Store Embeddings with Documents
const documents = [
async function prepareDocumentsWithEmbeddings() {
  const contents = documents.map(doc => doc.content);
  const embeddings = await getEmbeddings(contents);
  documents.forEach((doc, index) => {
    doc.embedding = embeddings[index];
  });
  // console.log('Documents with embeddings:', documents);
}
await prepareDocumentsWithEmbeddings();
    // Remove the embedding field from the search results
    const resultsWithoutEmbeddings = nearestDocs.map(doc => {
      const { embedding, ...rest } = doc;
    });
    console.log('Cosine similarity results:', resultsWithoutEmbeddings);
    return resultsWithoutEmbeddings;
  } else {
    // Remove the embedding field from the search results
    const resultsWithoutEmbeddings = results.map(result => {
      const doc = documents.find(doc => doc.id.toString() === result.ref);
    });
    console.log('Lunr search results:', resultsWithoutEmbeddings);
    return resultsWithoutEmbeddings;
  }

This is a "dumb" version of vector search, for prototyping RAG responses and UIs — with both regular search (w/ Lunr) and vector search (with OpenAI embeddings + cosine similarity)

This is an example of in-memory search, using a combination of lunr, OpenAI embeddings, and cosine similarity
Migrated from folder: Libraries/SemanticSearch/embeddingsSearchExample

Uses Val Town's [blob storage](https://docs.val.town/std/blob/) to search embeddings of all vals, by downloading them all and iterating through all of them to compute distance. Slow and terrible, but it works!

Uses [Neon](https://neon.tech/) to search embeddings of all vals, using the [pg_vector](https://neon.tech/docs/extensions/pgvector) extension.
- Call OpenAI to generate an embedding for the search query.
- Query the `vals_embeddings` table in Neon using the cosine similarity operator.
  - The `vals_embeddings` table gets refreshed every 10 minutes by [janpaul123/indexValsNeon](https://www.val.town/v/janpaul123/indexValsNeon).