Public
Script
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";
import { createClient } from "https://esm.sh/@libsql/client@0.6.0/web";
import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";
import OpenAI from "npm:openai";
export default async function semanticSearchPublicVals(query) {
const sqlite = createClient({
url: "libsql://valsembeddings-jpvaltown.turso.io",
authToken: Deno.env.get("TURSO_AUTH_TOKEN_VALSEMBEDDINGS"),
});
const openai = new OpenAI();
const embedding = await openai.embeddings.create({
model: "text-embedding-3-small",
input: query,
encoding_format: "base64",
dimensions: 256,
});
const embeddingBinary = base64Decode(embedding.data[0].embedding as any);
const res = await sqlToJSON(
await sqlite.execute({
sql:
"WITH matches AS (SELECT rowid, distance FROM vss_vals_embeddings WHERE vss_search(embedding, :embeddingBinary) LIMIT 50) SELECT id, distance FROM matches JOIN vals_embeddings ON matches.rowid = vals_embeddings.rowid",
args: { embeddingBinary },
}),
) as any;
const results = [];
for (const row of res) {
const [author_username, name, version] = row.id.split("!!");
results.push({ author_username, name, version, similarity: row.distance });
}
return results;
}
const exampleQuery = "check dynamicland website for changes and email me";
console.log(await semanticSearchPublicVals(exampleQuery));
Val Town is a social website to write and deploy JavaScript.
Build APIs and schedule functions from your browser.
May 29, 2024
This is really cool, what did you do to dump the vals into turso?
@nbbaier heh, still working on it :)
https://www.val.town/v/janpaul123/generateEmbeddingsForAllPublicVals
Oh cool, missed that val!
Yeah I only just made it public. It still has a bug causing not all vals to be indexed in the vss extension. Trying to figure that out now!