[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_107":108,"_109":110},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","cschick","name","design-database","path","scripts/find-duplicate-candidates.ts","project",{"_22":48,"_11":12,"_35":49,"_50":-5,"_51":44,"_52":53,"_54":53,"_55":56,"_57":-5,"_58":59,"_60":61,"_62":34,"_63":-5,"_64":-5,"_65":38,"_66":67,"_68":69,"_70":71,"_72":73,"_74":-5,"_75":76,"_77":76,"_78":79,"_80":79,"_81":79,"_82":76,"_83":76,"_84":85,"_86":87,"_88":89,"_90":44},"file",{"_22":23,"_11":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":-5,"_13":46},"permalink","https://www.val.town/x/cschick/design-database/code/scripts/find-duplicate-candidates.ts","extension","id","2ac99863-7bdf-4332-b8f1-6f593c6e8c9d","find-duplicate-candidates.ts","fileId","9bae891e-9c10-11f1-82b8-1607ee4eb77e","parentId","c47c07c1-2003-4f8b-8e73-b94386f5691d","branchId","f064a77f-da10-40b1-b2c8-85400a044673","branchVersion",30,"latest",true,"createdAt",["D",1787173057040],"version",0,"fileType","script","content","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nconst res = await sqlite.execute(`SELECT listing_id, title, image_hashes FROM listings`);\nconst listings = res.rows.map((r) => ({\n listing_id: r.listing_id as number,\n title: r.title as string,\n hashes: JSON.parse(r.image_hashes as string) as string[],\n}));\n\n// frequency of each image hash across all listings\nconst freq = new Map();\nfor (const l of listings) {\n for (const h of l.hashes) freq.set(h, (freq.get(h) ?? 0) + 1);\n}\n\nconsole.log(\"hash frequency distribution (freq -> count of hashes at that freq):\");\nconst distro = new Map();\nfor (const f of freq.values()) distro.set(f, (distro.get(f) ?? 0) + 1);\nconsole.log(JSON.stringify([...distro.entries()].sort((a, b) => a[0] - b[0])));\n\n// Boilerplate = images shared by many listings (care instructions, sizing chart, hand-drawn info, logo etc)\n// Rare-shared = images shared by a small handful of listings - likely the SAME design reused across listing variants\nconst BOILERPLATE_MIN = 15;\nconst rareShared = new Set([...freq.entries()].filter(([, c]) => c >= 2 && c < BOILERPLATE_MIN).map(([h]) => h));\n\n// meaningful (non-boilerplate) hash count per listing, for ratio calc\nconst meaningfulCount = new Map();\nfor (const l of listings) {\n meaningfulCount.set(l.listing_id, l.hashes.filter((h) => freq.get(h)! < BOILERPLATE_MIN).length);\n}\n\n// pairwise: for each rare-shared hash, connect the listings that share it\nconst pairShared = new Map(); // \"id1|id2\" (id1 shared rare hash count\nconst hashToListings = new Map();\nfor (const l of listings) {\n for (const h of l.hashes) {\n if (!rareShared.has(h)) continue;\n if (!hashToListings.has(h)) hashToListings.set(h, []);\n hashToListings.get(h)!.push(l.listing_id);\n }\n}\nfor (const ids of hashToListings.values()) {\n for (let i = 0; i < ids.length; i++) {\n for (let j = i + 1; j < ids.length; j++) {\n const key = ids[i] < ids[j] ? `${ids[i]}|${ids[j]}` : `${ids[j]}|${ids[i]}`;\n pairShared.set(key, (pairShared.get(key) ?? 0) + 1);\n }\n }\n}\n\nconst titleById = new Map(listings.map((l) => [l.listing_id, l.title]));\nconst rows = [...pairShared.entries()].map(([key, shared]) => {\n const [a, b] = key.split(\"|\").map(Number);\n const ratio = shared / Math.min(meaningfulCount.get(a)!, meaningfulCount.get(b)!);\n return { a, b, shared, ratio, titleA: titleById.get(a), titleB: titleById.get(b) };\n});\nrows.sort((x, y) => y.ratio - x.ratio || y.shared - x.shared);\n\nconsole.log(\"total candidate pairs (sharing >=1 rare image):\", rows.length);\nconst highConfidence = rows.filter((r) => r.ratio >= 0.5);\nconsole.log(\"high-confidence pairs (ratio >= 0.5):\", highConfidence.length);\nfor (const r of highConfidence) {\n console.log(`${(r.ratio * 100).toFixed(0)}% shared=${r.shared} | ${r.a} \"${r.titleA}\" <-> ${r.b} \"${r.titleB}\"`);\n}\n","deleted",false,"label",[47,24],"scripts","7615b982-4ecc-4be8-ba7a-c3d6aed0dd32",["D",1787169600594],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Offline design DB for Grey and Slate catalog","imageUrl","userId","442cb420-422c-11f0-bbe8-8f3f0ddc6612","pinnedFileId","0a7ef8ee-9c0a-11f1-bbe1-1607ee4eb77e","pinPreview","pinMigrated","screenshotVersion","likeCount","orgId","675fd955-734f-4afa-a1fc-de0c0cf86139","tursoName","7615b9824ecc4be8ba7ac3d6aed0dd32","tursoHostname","7615b9824ecc4be8ba7ac3d6aed0dd32-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789517443998],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_106":102},"org",{"_9":10,"_100":-5,"_101":102,"_58":59,"_103":104,"_105":44},"branch",{"_37":32,"_11":91,"_22":30,"_92":48,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":12,"_99":10},"isOwned","main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yeTY0alNTUThEcXp4czVhYzlySDZIeFBNa1kifQ","tier","free","canRestrictVals","profileImageUrl","ctx",{"_84":-5,"_140":141,"_142":143,"_144":145,"_146":44,"_147":44,"_148":44,"_149":34,"_150":-5,"_151":-7,"_152":-7,"_153":38,"_154":44,"_155":44,"_156":-5,"_157":44,"_158":44,"_159":44,"_160":-5},"ENV",{"_111":112,"_113":114,"_115":91,"_116":117,"_118":119,"_120":121,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135,"_136":137,"_138":139},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]