[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_163":164,"_165":166},"routes/_app.x.$handle.$name",{"_94":93,"_32":44,"_102":84,"_27":103,"_13":14,"_104":-5,"_11":105,"_106":107,"_108":109,"_110":111,"_112":113,"_20":-5,"_114":-5,"_115":-5,"_116":25,"_117":118,"_119":25,"_120":-5,"_121":-5,"_122":123},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":-5,"_56":48,"_57":-5,"_58":-5,"_59":60,"_61":62,"_63":-5,"_64":-5,"_65":-5,"_66":-5,"_67":68,"_69":68,"_70":71,"_72":71,"_73":71,"_74":68,"_75":68,"_76":77,"_78":79,"_80":81,"_82":25},"version",48,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","5fff8004-252d-11f0-8a67-569c3dd06744","name","fileType","script","content","import { createClient, InsertResult } from \"npm:@clickhouse/client\";\nimport { z } from \"npm:zod\";\nimport { format, getUnixTime } from 'npm:date-fns';\n\nconst schemaVersion = 3;\n\nfunction _client() {\n return createClient({\n url: Deno.env.get(\"WIDE_CLICKHOUSE_URL\"),\n username: \"default\",\n password: Deno.env.get(\"WIDE_CLICKHOUSE_PW\"),\n });\n}\n\nexport async function runMigrations() {\n for (\n const migration of [\n `\nCREATE TABLE IF NOT EXISTS logs_${schemaVersion}\n(\n _tenantId LowCardinality(String) NOT NULL CODEC(ZSTD(1)),\n _timestamp DateTime64(3) NOT NULL CODEC(DoubleDelta, LZ4),\n _now DateTime64(3) NOT NULL CODEC(ZSTD(1)),\n _ttl DateTime NOT NULL,\n _traceId String CODEC(ZSTD(1)),\n _requestId String CODEC(ZSTD(1)),\n _spanId String CODEC(ZSTD(1)),\n _coldStart UInt32 Codec(T64, LZ4),\n _parentSpanId String CODEC(ZSTD(1)),\n _startTime DateTime64(3) CODEC(DoubleDelta, LZ4),\n _endTime DateTime64(3) CODEC(DoubleDelta, LZ4),\n _duration UInt32 Codec(T64, LZ4),\n _source String NOT NULL CODEC(ZSTD(1)),\n \\`string.names\\` Array(String) CODEC(ZSTD(2)),\n \\`string.values\\` Array(String) CODEC(ZSTD(2)),\n \\`number.names\\` Array(String) CODEC(ZSTD(2)),\n \\`number.values\\` Array(Float64) CODEC(ZSTD(2)),\n \\`boolean.names\\` Array(String) CODEC(ZSTD(2)),\n \\`boolean.values\\` Array(UInt8) CODEC(ZSTD(2)),\n INDEX idx_tenant_bloom _tenantId TYPE bloom_filter GRANULARITY 4\n) ENGINE = MergeTree()\n ORDER BY (_now)`,\n ]\n ) {\n try {\n await _client().command({\n query: migration,\n });\n } catch (e) {\n console.error(migration, e);\n throw new Error(`Migration failed: ${e}`);\n }\n }\n}\nexport function parseRows(rows: any[]) {\n const processedRows = rows.map(row => {\n const paths: {\n string: Record;\n number: Record;\n boolean: Record;\n } = {\n string: {},\n number: {},\n boolean: {},\n };\n\n function processObject(obj: any, currentPath = \"\") {\n for (const [key, value] of Object.entries(obj)) {\n const newPath = currentPath ? `${currentPath}.${key}` : key;\n\n if (Array.isArray(value)) {\n continue;\n }\n\n if (typeof value === \"object\" && value !== null) {\n processObject(value, newPath);\n } else {\n const type = typeof value;\n if (type === \"string\" || type === \"number\" || type === \"boolean\") {\n paths[type][newPath] = value as any;\n }\n }\n }\n }\n\n processObject(row);\n return paths;\n });\n\n return processedRows;\n}\n\n// const rows = parseRows([\n// {\n// \"name\": \"John\",\n// \"age\": 30,\n// \"isActive\": true,\n// \"contact\": {\n// \"email\": \"john@example.com\",\n// \"phone\": {\n// \"home\": \"555-1234\",\n// \"mobile\": \"555-5678\",\n// },\n// },\n// \"scores\": [90, 85, 88], // This array will be ignored\n// \"metadata\": {\n// \"lastLogin\": \"2023-01-01\",\n// \"visits\": 42,\n// \"preferences\": {\n// \"darkMode\": true,\n// \"notifications\": false,\n// },\n// },\n// },\n// {\n// \"name\": \"Jane\",\n// \"age\": 28,\n// \"isActive\": false,\n// \"contact\": {\n// \"email\": \"jane@example.com\",\n// \"phone\": {\n// \"home\": \"555-4321\",\n// \"work\": \"555-8765\",\n// },\n// },\n// \"tags\": [\"developer\", \"designer\"], // This array will be ignored\n// \"metadata\": {\n// \"lastLogin\": \"2023-01-02\",\n// \"visits\": 37,\n// \"preferences\": {\n// \"darkMode\": false,\n// \"notifications\": true,\n// },\n// },\n// },\n// ]);\n\nconst writeRowsToClickhouse = async (userId: string, rows: ReturnType): Promise => {\n // Write to ClickHouse\n const now = new Date();\n const ttl = new Date(now.getTime() + (1 * 24 * 60 * 60 * 1000)); // 1 day TTL\n\n const clickhouseRows = rows.map(row => ({\n _tenantId: userId,\n _timestamp: now,\n _now: now,\n _ttl: ttl,\n \"string.names\": Object.keys(row.string),\n \"string.values\": Object.values(row.string),\n \"number.names\": Object.keys(row.number),\n \"number.values\": Object.values(row.number),\n \"boolean.names\": Object.keys(row.boolean),\n \"boolean.values\": Object.values(row.boolean),\n }));\n\n return await _client().insert({\n table: `logs_${schemaVersion}`,\n values: clickhouseRows,\n format: \"JSONEachRow\",\n });\n};\n\n// await writeRowsToClickhouse(rows);\n\n/**\n * write!\n */\nexport async function write(userId: string, data: any[]) {\n return await writeRowsToClickhouse(userId, parseRows(data));\n}\n\n/**\n * fields types\n */\nexport enum FieldType {\n String = \"string\",\n Number = \"number\",\n Boolean = \"boolean\",\n}\n\n/**\n * values!\n */\nexport async function values(\n userId: string,\n { field_name, field_type }: { field_name: string; field_type: FieldType },\n) {\n if (field_type !== FieldType.Boolean && field_type !== FieldType.Number && field_type !== FieldType.String) {\n throw new Error(`Invalid field type: ${field_type}`);\n }\n const queryString = `SELECT DISTINCT\n \\`${field_type}.values\\`[arrayFirst(i ->\n \\`${field_type}.names\\`[i] = {field_name: String},\n arrayEnumerate(\\`${field_type}.names\\`)\n )] as value,\n count(*) as count\nFROM logs_${schemaVersion}\nWHERE _tenantId = {userId: String}\nAND arrayExists(i ->\n \\`${field_type}.names\\`[i] = {field_name: String},\n arrayEnumerate(\\`${field_type}.names\\`)\n)\nGROUP BY value\nORDER BY count DESC; `;\n console.log(queryString, { field_name, userId });\n const query = await _client().query({\n query: queryString,\n format: \"JSONEachRow\",\n query_params: { field_name, userId },\n });\n return ((await query.json()).map((r: any) => ({ ...r, count: Number(r.count) })) as {\n value: string;\n count: number;\n }[]);\n}\n\n/**\n * fields!\n */\nexport async function fields(userId: string, search: string = \"\") {\n return (await (await _client().query({\n query: `\nSELECT \n name AS field_name,\n field_type,\n count() AS count\nFROM \n(\n -- String fields\n SELECT \n arrayJoin(\\`string.names\\`) AS name,\n 'string' AS field_type\n FROM logs_${schemaVersion}\n WHERE name ILIKE {name: String}\n AND _tenantId = {userId: String}\n \n UNION ALL\n \n -- Number fields\n SELECT \n arrayJoin(\\`number.names\\`) AS name,\n 'number' AS field_type\n FROM logs_${schemaVersion}\n WHERE name ILIKE {name: String}\n AND _tenantId = {userId: String}\n \n UNION ALL\n \n -- Boolean fields\n SELECT \n arrayJoin(\\`boolean.names\\`) AS name,\n 'boolean' AS field_type\n FROM logs_${schemaVersion}\n WHERE name ILIKE {name: String}\n AND _tenantId = {userId: String}\n)\nGROUP BY field_name, field_type\n`,\n format: \"JSONEachRow\",\n query_params: { name: search, userId },\n })).json() as any[]).map(r => ({ ...r, count: Number(r.count) })) as {\n field_name: string;\n field_type: FieldType;\n count: number;\n }[];\n}\n\nexport const StringOperator = z.enum([\"equals\", \"contains\", \"startsWith\", \"endsWith\"]);\nexport const NumberOperator = z.enum([\"equals\", \"greaterThan\", \"lessThan\", \"greaterThanOrEqual\", \"lessThanOrEqual\"]);\nexport const BooleanOperator = z.enum([\"equals\"]);\n\nexport const SearchFilter = z.union([\n z.object({\n fieldName: z.string(),\n value: z.string(),\n operator: StringOperator,\n }),\n z.object({\n fieldName: z.string(),\n value: z.number(),\n operator: NumberOperator,\n }),\n z.object({\n fieldName: z.string(),\n value: z.boolean(),\n operator: BooleanOperator,\n }),\n]);\n\nexport type TStringOperator = z.infer;\nexport type TNumberOperator = z.infer;\nexport type TBooleanOperator = z.infer;\nexport type TSearchFilter = z.infer;\n\n/**\n * Get the ClickHouse comparison expression for the given operator\n */\nfunction getComparisonExpression(\n operator: TStringOperator | TNumberOperator | TBooleanOperator,\n valueRef: string,\n): string {\n switch (operator) {\n // String operators\n case \"equals\":\n return `= ${valueRef}`;\n case \"contains\":\n return `LIKE concat('%', ${valueRef}, '%')`;\n case \"startsWith\":\n return `LIKE concat(${valueRef}, '%')`;\n case \"endsWith\":\n return `LIKE concat('%', ${valueRef})`;\n // Number operators\n case \"greaterThan\":\n return `> ${valueRef}`;\n case \"lessThan\":\n return `< ${valueRef}`;\n case \"greaterThanOrEqual\":\n return `>= ${valueRef}`;\n case \"lessThanOrEqual\":\n return `<= ${valueRef}`;\n default:\n throw new Error(`Unsupported operator: ${operator}`);\n }\n}\n\n/**\n * Enhanced search function supporting multiple conditions with different operators\n */\nexport async function search(userId: string, {\n filters,\n start,\n end,\n limit = 100,\n}: {\n filters: TSearchFilter[];\n start: Date | string;\n end: Date | string;\n limit?: number;\n}) {\n // Build the WHERE clause for each condition\n const whereConditions = filters.map((filter, index) => {\n let typeCondition: string;\n let paramType: string;\n\n if (typeof filter.value === \"string\") {\n typeCondition = \"string\";\n paramType = \"String\";\n } else if (typeof filter.value === \"number\") {\n typeCondition = \"number\";\n paramType = \"Float64\";\n } else if (typeof filter.value === \"boolean\") {\n typeCondition = \"boolean\";\n paramType = \"UInt8\";\n } else {\n throw new Error(`Unsupported value type: ${typeof (filter as any).value}`);\n }\n\n const comparisonExpr = getComparisonExpression(\n filter.operator,\n `{value${index}: ${paramType}}`,\n );\n\n return `arrayExists(i ->\n \\`${typeCondition}.names\\`[i] = {field_name${index}: String}\n AND \\`${typeCondition}.values\\`[i] ${comparisonExpr},\n arrayEnumerate(\\`${typeCondition}.names\\`)\n )`;\n });\n\n // Build the query\n const queryString = `\n SELECT\n \\`string.names\\`, \\`string.values\\`,\n \\`number.names\\`, \\`number.values\\`,\n \\`boolean.names\\`, \\`boolean.values\\`\n FROM logs_${schemaVersion}\n WHERE _tenantId = {userId: String}\n ${whereConditions.length > 0 ? `AND ${whereConditions.join(\" AND \")}` : \"\"}\n AND _timestamp >= {start: DateTime64}\n AND _timestamp < {end: DateTime64}\n limit {limit: UInt64}\n `;\n // Build query parameters\n const queryParams: Record = { userId };\n filters.forEach((filter, index) => {\n queryParams[`field_name${index}`] = filter.fieldName;\n queryParams[`value${index}`] = typeof filter.value === \"boolean\"\n ? Number(filter.value)\n : filter.value;\n });\n\n const query = await _client().query({\n query: queryString,\n format: \"JSONEachRow\",\n query_params: {\n ...queryParams,\n limit,\n // Best effort does not seem to work so we do this format dance, generally the api layer validates iso date formats so we should generally be processing those.\n start: formatDate(typeof start === \"string\" ? new Date(start) : start),\n end: formatDate(typeof end === \"string\" ? new Date(end) : end),\n },\n clickhouse_settings: {\n date_time_input_format: \"best_effort\",\n date_time_output_format: \"iso\",\n },\n });\n\n const results = await query.json() as LogRow[];\n return reconstructObjects(results);\n}\n\n\nfunction formatDate(date: Date) {\n return String(getUnixTime(date) + (date.getMilliseconds() / 1000))\n}\n\ninterface LogRow {\n \"string.names\": string[];\n \"string.values\": string[];\n \"number.names\": string[];\n \"number.values\": number[];\n \"boolean.names\": string[];\n \"boolean.values\": boolean[];\n}\n\nfunction reconstructObjects(\n results: LogRow[],\n) {\n // Reconstruct the original objects from the columnar data\n return results.map((row) => {\n const reconstructed: Record = {};\n\n // Helper function to set nested object value\n const setNestedValue = (obj: any, path: string, value: any) => {\n const parts = path.split(\".\");\n let current = obj;\n\n for (let i = 0; i < parts.length - 1; i++) {\n if (!(parts[i] in current)) {\n current[parts[i]] = {};\n }\n current = current[parts[i]];\n }\n\n current[parts[parts.length - 1]] = value;\n };\n\n // Process string fields\n for (let i = 0; i < row[\"string.names\"].length; i++) {\n setNestedValue(reconstructed, row[\"string.names\"][i], row[\"string.values\"][i]);\n }\n\n // Process number fields\n for (let i = 0; i < row[\"number.names\"].length; i++) {\n setNestedValue(reconstructed, row[\"number.names\"][i], row[\"number.values\"][i]);\n }\n\n // Process boolean fields\n for (let i = 0; i < row[\"boolean.names\"].length; i++) {\n setNestedValue(reconstructed, row[\"boolean.names\"][i], Boolean(row[\"boolean.values\"][i]));\n }\n\n return reconstructed;\n });\n}","createdAt",["D",1732722301288],"branchVersion","branchId","5ffd17ba-252d-11f0-8a67-569c3dd06744","id","5ffccbb6-252d-11f0-8a67-569c3dd06744","wideLib",["D",1745953726262],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","418c1e39-eb48-4f2f-8c97-122ece94631c","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","c1e7642a-74cc-4d69-8652-20abbdeaa94e","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_94":93,"_101":97},"org",{"_94":93,"_95":-5,"_96":97,"_18":54,"_98":99,"_100":25},"branch",{"_13":83,"_32":84,"_42":41,"_85":43,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":44,"_92":93},"isOwned",54,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","maxm","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJnTEtkSHlOMWxWZm1GRHdKYUVTNklYM3FpdiJ9","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_42":43,"_32":44,"_37":159,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":-5,"_56":48,"_57":-5,"_58":-5,"_59":60,"_61":62,"_63":-5,"_64":-5,"_65":-5,"_66":-5,"_67":68,"_69":68,"_70":71,"_72":71,"_73":71,"_74":68,"_75":68,"_76":160,"_78":161,"_80":162,"_82":25},"tree",[150,151],"files",[150,151],"counts",{"_140":60,"_141":136,"_142":60,"_143":144,"_145":146},"contents",[124,125],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":133,"_32":134,"_30":135,"_127":-5,"_40":41,"_39":136,"_128":48,"_37":137,"_13":60,"_33":20,"_35":138,"_24":25,"_131":-5,"_27":139},{"_42":126,"_32":28,"_30":31,"_127":-5,"_40":41,"_39":14,"_128":25,"_37":129,"_13":130,"_33":34,"_35":36,"_24":25,"_131":-5,"_27":132},"610f7f3a-252d-11f0-8a67-569c3dd06744","parentId","latest",["D",1732722301288],47,"label",[28],"5fff8388-252d-11f0-8a67-569c3dd06744","README.md","5fff83f6-252d-11f0-8a67-569c3dd06744",1,["D",1732392026816],"\n\nMigrated from folder: wide/wideLib",[134],"pulls","branches","forks","envVars",{"_147":60,"_149":60},"connections",{"_147":60,"_148":60},"count","unfilledCount","emptyCount",{"_42":133,"_32":134,"_30":135,"_127":-5,"_40":41,"_39":136,"_128":48,"_37":156,"_13":60,"_33":20,"_35":138,"_24":25,"_131":-5,"_153":157,"_27":158},{"_42":126,"_32":28,"_30":31,"_127":-5,"_40":41,"_39":14,"_128":25,"_37":152,"_13":130,"_33":34,"_35":36,"_24":25,"_131":-5,"_153":154,"_27":155},["D",1732722301288],"children",[],[28],["D",1732392026816],[],[134],["D",1745953726262],{"_94":93,"_101":97},{"_94":93,"_95":-5,"_96":97,"_18":54,"_98":99,"_100":25},{"_13":83,"_32":84,"_42":41,"_85":43,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":44,"_92":93},"ctx",{"_76":-5,"_121":196,"_197":198,"_199":200,"_201":25,"_202":25,"_203":25,"_204":48,"_205":-5,"_206":-7,"_207":-7,"_208":60,"_209":25,"_210":25,"_211":-5,"_212":25,"_213":25,"_214":25,"_215":-5},"ENV",{"_167":168,"_169":170,"_171":84,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e8c4b7030cb7600e4eb0736b15f422c8b8bd130f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v48