diff --git a/package-lock.json b/package-lock.json index 0110bb4e2ba..e1f9a863ff2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13590,6 +13590,16 @@ "integrity": "sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==", "dev": true }, + "node_modules/@types/mongodb": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-4.0.6.tgz", + "integrity": "sha512-XTbn1Z1j7fHzC1Vkd9LYO48lO2C581r+oRCi/KNzcTHIri7hEaya8r9vxoHJiKr+oeUWVK69+9xr84Mp+aReaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mongodb": "*" + } + }, "node_modules/@types/ms": { "version": "0.7.32", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.32.tgz", @@ -17634,6 +17644,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsite": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", @@ -21216,6 +21239,20 @@ "node": ">= 0.8.0" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/dup": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dup/-/dup-1.0.0.tgz", @@ -22370,12 +22407,10 @@ "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -22418,9 +22453,10 @@ "integrity": "sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==" }, "node_modules/es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", "dependencies": { "es-errors": "^1.3.0" }, @@ -22429,13 +22465,15 @@ } }, "node_modules/es-set-tostringtag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", - "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.2.4", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", - "hasown": "^2.0.1" + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -24731,6 +24769,15 @@ "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==", "license": "MIT" }, + "node_modules/formdata-node": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", + "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -25249,15 +25296,21 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -25501,6 +25554,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -26195,11 +26261,12 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -26475,9 +26542,10 @@ } }, "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -28570,6 +28638,12 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-base64": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz", + "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==", + "license": "BSD-3-Clause" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -32137,6 +32211,15 @@ "integrity": "sha512-c4vLwYWyl+Ji+U43eU/G5FwxWd4ZH0ePUsFs5y0uwD9HUEFBXUQ1zUUan+78IpRD+y4pUfG0nAzNM292K7ItvA==", "dev": true }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/math-log2": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/math-log2/-/math-log2-1.0.1.tgz", @@ -43918,6 +44001,12 @@ "querystring": "0.2.0" } }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -44105,6 +44194,141 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, + "node_modules/voyageai": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/voyageai/-/voyageai-0.0.4.tgz", + "integrity": "sha512-eHSwflQdhByXpudW49LNEjJr1cpz9GmTFADiGr+b5TgwNgewiRjJrgDS3X/s1wExCdAUXW+az1DrF9pfHrPyyA==", + "dependencies": { + "form-data": "^4.0.0", + "formdata-node": "^6.0.3", + "js-base64": "3.7.2", + "node-fetch": "2.7.0", + "qs": "6.11.2", + "readable-stream": "^4.5.2", + "url-join": "4.0.1" + } + }, + "node_modules/voyageai/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/voyageai/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/voyageai/node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/voyageai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/voyageai/node_modules/qs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/voyageai/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/voyageai/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/voyageai/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/voyageai/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/vt-pbf": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/vt-pbf/-/vt-pbf-3.1.3.tgz", @@ -49727,9 +49951,11 @@ "@leafygreen-ui/tooltip": "^13.0.12", "@types/plotly.js": "^3.0.0", "ml-pca": "^4.1.1", + "mongodb": "^6.16.0", "plotly.js": "^3.0.1", "react": "^17.0.2", - "react-dom": "^17.0.2" + "react-dom": "^17.0.2", + "voyageai": "^0.0.4" }, "devDependencies": { "@mongodb-js/eslint-config-compass": "^1.3.8", @@ -49740,6 +49966,7 @@ "@types/chai": "^4.2.21", "@types/chai-dom": "^0.0.10", "@types/mocha": "^9.0.0", + "@types/mongodb": "^4.0.6", "@types/react": "^17.0.5", "@types/react-dom": "^17.0.10", "@types/sinon-chai": "^3.2.5", @@ -49749,7 +49976,7 @@ "mocha": "^10.2.0", "nyc": "^15.1.0", "sinon": "^17.0.1", - "typescript": "^5.0.4", + "typescript": "^5.8.3", "xvfb-maybe": "^0.2.1" } }, @@ -49896,6 +50123,20 @@ "url": "https://opencollective.com/sinon" } }, + "packages/compass-vector-embedding-visualizer/node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "packages/compass-web": { "name": "@mongodb-js/compass-web", "version": "0.16.0", @@ -61813,6 +62054,7 @@ "@types/chai": "^4.2.21", "@types/chai-dom": "^0.0.10", "@types/mocha": "^9.0.0", + "@types/mongodb": "^4.0.6", "@types/plotly.js": "^3.0.0", "@types/react": "^17.0.5", "@types/react-dom": "^17.0.10", @@ -61822,12 +62064,14 @@ "hadron-app-registry": "^9.4.8", "ml-pca": "^4.1.1", "mocha": "^10.2.0", + "mongodb": "^6.16.0", "nyc": "^15.1.0", "plotly.js": "^3.0.1", "react": "^17.0.2", "react-dom": "^17.0.2", "sinon": "^17.0.1", - "typescript": "^5.0.4", + "typescript": "^5.8.3", + "voyageai": "^0.0.4", "xvfb-maybe": "^0.2.1" }, "dependencies": { @@ -61950,6 +62194,12 @@ "nise": "^5.1.5", "supports-color": "^7.2.0" } + }, + "typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true } } }, @@ -67595,6 +67845,15 @@ "integrity": "sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==", "dev": true }, + "@types/mongodb": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-4.0.6.tgz", + "integrity": "sha512-XTbn1Z1j7fHzC1Vkd9LYO48lO2C581r+oRCi/KNzcTHIri7hEaya8r9vxoHJiKr+oeUWVK69+9xr84Mp+aReaw==", + "dev": true, + "requires": { + "mongodb": "*" + } + }, "@types/ms": { "version": "0.7.32", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.32.tgz", @@ -70863,6 +71122,15 @@ "set-function-length": "^1.2.1" } }, + "call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + } + }, "callsite": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", @@ -73998,6 +74266,16 @@ "resolved": "https://registry.npmjs.org/dtype/-/dtype-2.0.0.tgz", "integrity": "sha512-s2YVcLKdFGS0hpFqJaTwscsyt0E8nNFdmo73Ocd81xNPj4URI4rj6D60A+vFMIw7BXWlb4yRkEwfBqcZzPGiZg==" }, + "dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "requires": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + } + }, "dup": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dup/-/dup-1.0.0.tgz", @@ -74900,12 +75178,9 @@ "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" }, "es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "requires": { - "get-intrinsic": "^1.2.4" - } + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" }, "es-errors": { "version": "1.3.0", @@ -74941,21 +75216,22 @@ "integrity": "sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==" }, "es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "requires": { "es-errors": "^1.3.0" } }, "es-set-tostringtag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", - "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", "requires": { - "get-intrinsic": "^1.2.4", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", - "hasown": "^2.0.1" + "hasown": "^2.0.2" } }, "es-to-primitive": { @@ -76725,6 +77001,11 @@ "resolved": "https://registry.npmjs.org/format-util/-/format-util-1.0.5.tgz", "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==" }, + "formdata-node": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", + "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==" + }, "formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -77118,15 +77399,20 @@ "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==" }, "get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "requires": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" } }, "get-package-info": { @@ -77305,6 +77591,15 @@ "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", "dev": true }, + "get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "requires": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + } + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -77896,12 +78191,9 @@ } }, "gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "requires": { - "get-intrinsic": "^1.1.3" - } + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" }, "got": { "version": "10.7.0", @@ -78908,9 +79200,9 @@ "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==" }, "has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" }, "has-to-string-tag-x": { "version": "1.4.1", @@ -80396,6 +80688,11 @@ "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz", "integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==" }, + "js-base64": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz", + "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -83340,6 +83637,11 @@ "integrity": "sha512-c4vLwYWyl+Ji+U43eU/G5FwxWd4ZH0ePUsFs5y0uwD9HUEFBXUQ1zUUan+78IpRD+y4pUfG0nAzNM292K7ItvA==", "dev": true }, + "math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, "math-log2": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/math-log2/-/math-log2-1.0.1.tgz", @@ -92747,6 +93049,11 @@ } } }, + "url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==" + }, "url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -92903,6 +93210,94 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, + "voyageai": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/voyageai/-/voyageai-0.0.4.tgz", + "integrity": "sha512-eHSwflQdhByXpudW49LNEjJr1cpz9GmTFADiGr+b5TgwNgewiRjJrgDS3X/s1wExCdAUXW+az1DrF9pfHrPyyA==", + "requires": { + "form-data": "^4.0.0", + "formdata-node": "^6.0.3", + "js-base64": "3.7.2", + "node-fetch": "2.7.0", + "qs": "6.11.2", + "readable-stream": "^4.5.2", + "url-join": "4.0.1" + }, + "dependencies": { + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, + "form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + } + }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "qs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", + "requires": { + "side-channel": "^1.0.4" + } + }, + "readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "requires": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, "vt-pbf": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/vt-pbf/-/vt-pbf-3.1.3.tgz", diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index 3a5182ac913..0a181d2acad 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -2632,6 +2632,7 @@ type ScreenEvent = ConnectionScopedEvent<{ | 'my_queries' | 'performance' | 'schema' + | 'vector_visualizer' | 'validation' | 'confirm_new_pipeline_modal' | 'create_collection_modal' diff --git a/packages/compass-vector-embedding-visualizer/package.json b/packages/compass-vector-embedding-visualizer/package.json index a532309d56d..0784a277500 100644 --- a/packages/compass-vector-embedding-visualizer/package.json +++ b/packages/compass-vector-embedding-visualizer/package.json @@ -52,9 +52,11 @@ "@leafygreen-ui/tooltip": "^13.0.12", "@types/plotly.js": "^3.0.0", "ml-pca": "^4.1.1", + "mongodb": "^6.16.0", "plotly.js": "^3.0.1", "react": "^17.0.2", - "react-dom": "^17.0.2" + "react-dom": "^17.0.2", + "voyageai": "^0.0.4" }, "devDependencies": { "@mongodb-js/eslint-config-compass": "^1.3.8", @@ -65,6 +67,7 @@ "@types/chai": "^4.2.21", "@types/chai-dom": "^0.0.10", "@types/mocha": "^9.0.0", + "@types/mongodb": "^4.0.6", "@types/react": "^17.0.5", "@types/react-dom": "^17.0.10", "@types/sinon-chai": "^3.2.5", @@ -74,7 +77,7 @@ "mocha": "^10.2.0", "nyc": "^15.1.0", "sinon": "^17.0.1", - "typescript": "^5.0.4", + "typescript": "^5.8.3", "xvfb-maybe": "^0.2.1" }, "is_compass_plugin": true diff --git a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx index db17bbff4d1..380ddd0f9e3 100644 --- a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx +++ b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx @@ -1,97 +1,203 @@ import React, { useEffect, useState } from 'react'; +import { connect } from 'react-redux'; import Plotly from 'plotly.js'; +import * as PCA from 'ml-pca'; +import { Binary } from 'mongodb'; +import type { Document } from 'bson'; -type HoverInfo = { - x: number; - y: number; - text: string; -} | null; +import type { VectorEmbeddingVisualizerState } from '../stores/reducer'; +import { loadDocuments, runVectorAggregation } from '../stores/visualization'; +import { ErrorSummary, SpinLoader } from '@mongodb-js/compass-components'; -export const VectorVisualizer: React.FC = () => { +type HoverInfo = { x: number; y: number; text: string } | null; + +export interface VectorVisualizerProps { + onFetchDocs: () => void; + onFetchAgg: () => void; + docs: Document[]; + aggResults: { candidates: Document[]; limited: Document[] }; + loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; + loadingDocumentsError: Error | null; +} + +function normalizeTo2D(vectors: Binary[]): { x: number; y: number }[] { + const raw = vectors.map((v) => Array.from(v.toFloat32Array())); + const pca = new PCA.PCA(raw); + const reduced = pca.predict(raw, { nComponents: 2 }).to2DArray(); + return reduced.map(([x, y]) => ({ x, y })); +} + +const VectorVisualizer: React.FC = ({ + onFetchDocs, + onFetchAgg, + docs, + aggResults, + loadingDocumentsState, + loadingDocumentsError, +}) => { const [hoverInfo, setHoverInfo] = useState(null); + const [query, setQuery] = useState(''); + const [shouldPlot, setShouldPlot] = useState(false); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (loadingDocumentsState === 'initial') { + onFetchDocs(); + } + }, [loadingDocumentsState, onFetchDocs]); useEffect(() => { + if (query) { + onFetchAgg(); + setLoading(true); + const timeout = setTimeout(() => { + setShouldPlot(true); + setLoading(false); + }, 600); + return () => clearTimeout(timeout); + } + }, [query, onFetchAgg]); + + useEffect(() => { + if (!shouldPlot) return; + const container = document.getElementById('vector-plot'); if (!container) return; - let isMounted = true; + const abortController = new AbortController(); const plot = async () => { - await Plotly.newPlot( - container, - [ - { - x: [1, 2, 3, 4, 5], - y: [10, 15, 13, 17, 12], - mode: 'markers', - type: 'scatter', - name: 'baskd', - text: ['doc1', 'doc2', 'doc3', 'doc4', 'doc5'], - hoverinfo: 'none', - marker: { - size: 15, - color: 'teal', - line: { width: 1, color: '#fff' }, + try { + if (docs.length === 0) return; + + const points = normalizeTo2D( + docs + .map((doc) => doc.review_vec) + .filter(Boolean) + .slice(0, 500) + ); + + const candidateIds = new Set( + aggResults.candidates.map((doc) => doc._id.toString()) + ); + const limitedIds = new Set( + aggResults.limited.map((doc) => doc._id.toString()) + ); + + await Plotly.newPlot( + container, + [ + { + x: points.map((p) => p.x), + y: points.map((p) => p.y), + mode: 'markers', + type: 'scatter', + text: docs.map((doc) => { + const review = doc.review || '[no text]'; + return review.length > 50 + ? review.match(/.{1,50}/g)?.join('
') || review + : review; + }), + hoverinfo: 'text', + marker: { + size: 12, + color: docs.map((doc) => { + const hasLimitedId = limitedIds.has(doc._id.toString()); + const hasCandidateId = candidateIds.has(doc._id.toString()); + if (hasLimitedId) return 'red'; + if (hasCandidateId) return 'orange'; + return 'teal'; + }), + line: { width: 1, color: '#fff' }, + }, }, + ], + { + hovermode: 'closest', + margin: { l: 40, r: 10, t: 30, b: 30 }, + plot_bgcolor: '#f9f9f9', + paper_bgcolor: '#f9f9f9', }, - ], - { - margin: { l: 40, r: 10, t: 40, b: 40 }, - hovermode: 'closest', - hoverdistance: 30, - dragmode: 'zoom', - plot_bgcolor: '#f7f7f7', - paper_bgcolor: '#f7f7f7', - xaxis: { gridcolor: '#e0e0e0' }, - yaxis: { gridcolor: '#e0e0e0' }, - }, - { responsive: true } - ); - - const handleHover = (data: any) => { - const point = data.points?.[0]; - if (!point) return; - - const containerRect = container.getBoundingClientRect(); - const relX = data.event.clientX - containerRect.left; - const relY = data.event.clientY - containerRect.top; - - if (isMounted) { - setHoverInfo({ x: relX, y: relY, text: point.text }); - } - }; - - const handleUnhover = () => { - if (isMounted) { - setHoverInfo(null); - } - }; - - container.addEventListener('plotly_hover', handleHover); - container.addEventListener('plotly_unhover', handleUnhover); - - // Cleanup - return () => { - isMounted = false; - container.removeEventListener('plotly_hover', handleHover); - container.removeEventListener('plotly_unhover', handleUnhover); - }; + { + responsive: true, + displayModeBar: false, + } + ); + } catch (err) { + console.error('VectorVisualizer error:', err); + } }; - let cleanup: (() => void) | undefined; - void plot().then((c) => { - if (typeof c === 'function') cleanup = c; - }); + void plot(); return () => { - isMounted = false; - if (cleanup) cleanup(); + abortController.abort(); }; - }, []); + }, [docs, aggResults, shouldPlot]); + + const onInput = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + const inputQuery = e.currentTarget.value.trim(); + if (inputQuery) { + setQuery(inputQuery); + setShouldPlot(false); + } + } + }; return (
-
+
+ +
+ + {loading && ( +
+ +
+ )} + +
+ + {loadingDocumentsError && ( + + )} + {hoverInfo && (
{ padding: '4px 8px', borderRadius: 4, pointerEvents: 'none', - whiteSpace: 'nowrap', zIndex: 1000, + whiteSpace: 'nowrap', }} > {hoverInfo.text} @@ -113,3 +219,16 @@ export const VectorVisualizer: React.FC = () => {
); }; + +export default connect( + (state: VectorEmbeddingVisualizerState) => ({ + docs: state.visualization.docs, + aggResults: state.visualization.aggResults, + loadingDocumentsState: state.visualization.loadingDocumentsState, + loadingDocumentsError: state.visualization.loadingDocumentsError, + }), + { + onFetchDocs: loadDocuments, + onFetchAgg: runVectorAggregation, + } +)(VectorVisualizer); diff --git a/packages/compass-vector-embedding-visualizer/src/index.ts b/packages/compass-vector-embedding-visualizer/src/index.ts index 319f993ca36..05460167eba 100644 --- a/packages/compass-vector-embedding-visualizer/src/index.ts +++ b/packages/compass-vector-embedding-visualizer/src/index.ts @@ -1,15 +1,18 @@ -// plugin.tsx import React from 'react'; import { registerHadronPlugin } from 'hadron-app-registry'; import { createLoggerLocator } from '@mongodb-js/compass-logging/provider'; +import { collectionModelLocator } from '@mongodb-js/compass-app-stores/provider'; +import { + dataServiceLocator, + type DataServiceLocator, +} from '@mongodb-js/compass-connections/provider'; -import { VectorVisualizer } from './components/vector-visualizer'; -import { createStore } from 'redux'; - -// Minimal reducer for the plugin store -function reducer(state = {}, action: any) { - return state; -} +import VectorVisualizer from './components/vector-visualizer'; +import { VectorsTabTitle } from './plugin-tab-title'; +import { + activateVectorPlugin, + type VectorDataServiceProps, +} from './stores/store'; export const CompassVectorPluginProvider = registerHadronPlugin( { @@ -17,30 +20,28 @@ export const CompassVectorPluginProvider = registerHadronPlugin( component: function VectorVisualizerProvider({ children }) { return React.createElement(React.Fragment, null, children); }, - activate: () => { - const store = createStore(reducer); - return { - store: () => store, - deactivate: () => { - // ignore - }, - }; - }, + activate: activateVectorPlugin, }, { - // collection: collectionModelLocator, - // dataService: dataServiceLocator, + dataService: + dataServiceLocator as DataServiceLocator, + collection: collectionModelLocator, logger: createLoggerLocator('COMPASS-VECTOR-VISUALIZER'), - // track: telemetryLocator, } ); -export default CompassVectorPluginProvider; +// const VectorVisualizerWrapper = (props: { +// dataService: any; +// collection: any; +// }) => { +// return React.createElement(VectorVisualizer, props); +// }; export const CompassVectorPlugin = { - name: 'VectorVisualizer', - type: 'Collection' as const, + name: 'Vector Visualizer' as const, provider: CompassVectorPluginProvider, - content: VectorVisualizer, - header: () => React.createElement('div', null, 'Vector Embeddings'), + content: VectorVisualizer, // VectorVisualizerWrapper, + header: VectorsTabTitle, }; + +export default CompassVectorPluginProvider; diff --git a/packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx b/packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx new file mode 100644 index 00000000000..87d3cab7121 --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx @@ -0,0 +1,5 @@ +import React from 'react'; + +export function VectorsTabTitle() { + return
Vectors
; +} diff --git a/packages/compass-vector-embedding-visualizer/src/stores/reducer.ts b/packages/compass-vector-embedding-visualizer/src/stores/reducer.ts new file mode 100644 index 00000000000..b9c4b04bb70 --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/reducer.ts @@ -0,0 +1,35 @@ +import type { AnyAction } from 'redux'; +import { combineReducers } from 'redux'; +import type { + VisualizationActions, + VisualizationActionTypes, +} from './visualization'; +import type { ThunkAction } from 'redux-thunk'; +import { visualizationReducer } from './visualization'; +import type { VectorPluginServices } from './store'; + +const reducer = combineReducers({ + visualization: visualizationReducer, +}); + +export type VectorEmbeddingVisualizerActions = VisualizationActions; + +export type VectorEmbeddingVisualizerActionTypes = VisualizationActionTypes; + +export type VectorEmbeddingVisualizerState = ReturnType; + +export type VectorEmbeddingVisualizerExtraArgs = VectorPluginServices & { + cancelControllerRef: { current: AbortController | null }; +}; + +export type VectorEmbeddingVisualizerThunkAction< + R, + A extends AnyAction +> = ThunkAction< + R, + VectorEmbeddingVisualizerState, + VectorEmbeddingVisualizerExtraArgs, + A +>; + +export default reducer; diff --git a/packages/compass-vector-embedding-visualizer/src/stores/store.ts b/packages/compass-vector-embedding-visualizer/src/stores/store.ts new file mode 100644 index 00000000000..b6fc394826e --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/store.ts @@ -0,0 +1,64 @@ +import { applyMiddleware, createStore } from 'redux'; +import type { DataService } from 'mongodb-data-service'; +// import type { ConnectionInfoRef } from '@mongodb-js/compass-connections/provider'; +import type { + Collection, + // MongoDBInstance, +} from '@mongodb-js/compass-app-stores/provider'; +// import type AppRegistry from 'hadron-app-registry'; +import type { Logger } from '@mongodb-js/compass-logging'; +// import type { TrackFunction } from '@mongodb-js/compass-telemetry'; +// import type { AtlasService } from '@mongodb-js/atlas-service/provider'; +// import type { PreferencesAccess } from 'compass-preferences-model'; +import type { ActivateHelpers } from 'hadron-app-registry'; +import thunk from 'redux-thunk'; + +import reducer from './reducer'; + +export type VectorDataServiceProps = + | 'find' + | 'aggregate' + // Required for collection model (fetching stats) + | 'collectionStats' + | 'collectionInfo' + | 'listCollections'; +export type VectorDataService = Pick; + +export type VectorPluginServices = { + dataService: VectorDataService; + logger: Logger; + collection: Collection; + + // Note(Rhys): If we want more of these services, we can uncomment, + // and then in ../index.ts, we add them as well. + + // connectionInfoRef: ConnectionInfoRef; + // instance: MongoDBInstance; + // localAppRegistry: Pick; + // globalAppRegistry: Pick; + // track: TrackFunction; + // atlasService: AtlasService; + // preferences: PreferencesAccess; +}; + +// export type VectorPluginOptions = { +// namespace: string; +// serverVersion: string; +// isReadonly: boolean; +// }; +export type VectorPluginOptions = Record; + +export function activateVectorPlugin( + _options: VectorPluginOptions, + services: VectorPluginServices, + { cleanup }: ActivateHelpers +) { + const cancelControllerRef = { current: null }; + const store = createStore( + reducer, + applyMiddleware( + thunk.withExtraArgument({ ...services, cancelControllerRef }) + ) + ); + return { store, deactivate: cleanup }; +} diff --git a/packages/compass-vector-embedding-visualizer/src/stores/util.ts b/packages/compass-vector-embedding-visualizer/src/stores/util.ts new file mode 100644 index 00000000000..e680777376e --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/util.ts @@ -0,0 +1,12 @@ +import type { AnyAction } from 'redux'; +import type { + VectorEmbeddingVisualizerActions, + VectorEmbeddingVisualizerActionTypes, +} from './reducer'; + +export function isAction( + action: AnyAction, + type: T +): action is Extract { + return action.type === type; +} diff --git a/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts new file mode 100644 index 00000000000..2d1381854cf --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts @@ -0,0 +1,257 @@ +import type { Reducer } from 'redux'; +import type { Document } from 'bson'; +import { Binary } from 'mongodb'; + +import { isAction } from './util'; +import type { VectorEmbeddingVisualizerThunkAction } from './reducer'; +import { VectorDataService } from './store'; +import { VoyageAIClient } from 'voyageai'; // Adjust import as needed + +export type VisualizationState = { + loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; + loadingDocumentsError: Error | null; + docs: Document[]; + aggResults: { candidates: Document[]; limited: Document[] }; +}; + +export enum VisualizationActionTypes { + FETCH_DOCUMENTS_STARTED = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_STARTED', + FETCH_DOCUMENTS_SUCCESS = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_SUCCESS', + FETCH_DOCUMENTS_FAILED = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_FAILED', + FETCH_AGG_STARTED = 'vector-embedding-visualizer/visualization/FETCH_AGG_STARTED', + FETCH_AGG_SUCCESS = 'vector-embedding-visualizer/visualization/FETCH_AGG_SUCCESS', + FETCH_AGG_FAILED = 'vector-embedding-visualizer/visualization/FETCH_AGG_FAILED', +} + +export type FetchDocumentsStartedAction = { + type: VisualizationActionTypes.FETCH_DOCUMENTS_STARTED; +}; + +export type FetchDocumentsSuccessAction = { + type: VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS; + docs: Document[]; +}; + +export type FetchDocumentsFailedAction = { + type: VisualizationActionTypes.FETCH_DOCUMENTS_FAILED; + error: Error; +}; + +export type FetchAggStartedAction = { + type: VisualizationActionTypes.FETCH_AGG_STARTED; +}; + +export type FetchAggSuccessAction = { + type: VisualizationActionTypes.FETCH_AGG_SUCCESS; + aggResults: { candidates: Document[]; limited: Document[] }; +}; + +export type FetchAggFailedAction = { + type: VisualizationActionTypes.FETCH_AGG_FAILED; + error: Error; +}; + +export type VisualizationActions = + | FetchDocumentsStartedAction + | FetchDocumentsSuccessAction + | FetchDocumentsFailedAction + | FetchAggStartedAction + | FetchAggSuccessAction + | FetchAggFailedAction; + +const INITIAL_STATE: VisualizationState = { + loadingDocumentsState: 'initial', + loadingDocumentsError: null, + docs: [], + aggResults: { candidates: [], limited: [] }, +}; + +export const visualizationReducer: Reducer = ( + state = INITIAL_STATE, + action +) => { + if (isAction(action, VisualizationActionTypes.FETCH_DOCUMENTS_STARTED)) { + return { + ...state, + loadingDocumentsState: 'loading', + loadingDocumentsError: null, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS)) { + return { + ...state, + loadingDocumentsState: 'loaded', + docs: action.docs, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_DOCUMENTS_FAILED)) { + return { + ...state, + loadingDocumentsState: 'error', + loadingDocumentsError: action.error, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_AGG_STARTED)) { + return { + ...state, + loadingDocumentsError: null, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_AGG_SUCCESS)) { + return { + ...state, + aggResults: action.aggResults, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_AGG_FAILED)) { + return { + ...state, + loadingDocumentsError: action.error, + }; + } + return state; +}; + +export function loadDocuments(): VectorEmbeddingVisualizerThunkAction< + Promise, + | FetchDocumentsStartedAction + | FetchDocumentsSuccessAction + | FetchDocumentsFailedAction +> { + return async function fetchDocs( + dispatch, + getState, + { dataService, collection } + ) { + dispatch({ + type: VisualizationActionTypes.FETCH_DOCUMENTS_STARTED, + }); + + try { + const docs = await dataService.find( + `${collection.database}.${collection.name}`, + {}, + { limit: 1000 } + ); + + dispatch({ + type: VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS, + docs, + }); + } catch (err) { + dispatch({ + type: VisualizationActionTypes.FETCH_DOCUMENTS_FAILED, + error: err as Error, + }); + } + }; +} + +//@ts-expect-error: I knowwwwww +globalThis.vectorCache = new Map(); + +async function r( + ns: string, + collection: VectorDataService, + query: string, + numCandidates: number, + limit: number +) { + const voyage = new VoyageAIClient({ apiKey: process.env.VOYAGEAI_API_KEY }); + + // Try to get vector from cache + //@ts-expect-error: I knowwwwww + let vector = globalThis.vectorCache.get(query); + + if (vector == null) { + // Get vector from VoyageAI + console.log('Fetching vector from VoyageAI'); + const response = await voyage.embed({ + model: 'voyage-3-large', + input: query, + }); + + if ( + response.data == null || + response.data[0] == null || + response.data[0].embedding == null + ) { + throw new Error('No vector found'); + } + vector = Binary.fromFloat32Array( + new Float32Array(response.data[0].embedding) + ); + + // Cache the vector for future use + + //@ts-expect-error: I knowwwwww + globalThis.vectorCache.set(query, vector); + } else { + console.log('Fetching vector from Cache'); + } + + // Run vector search aggregation + const pipeline = (numCandidates: number, limit: number) => [ + { + $vectorSearch: { + index: 'real_for_real_index', + path: 'review_vec', + queryVector: vector, + numCandidates, + limit, + }, + }, + ]; + + const candidates = await collection.aggregate( + ns, + pipeline(numCandidates, numCandidates) + ); + + const limited = await collection.aggregate( + ns, + pipeline(numCandidates, limit) + ); + + return { candidates, limited }; +} + +export function runVectorAggregation(): VectorEmbeddingVisualizerThunkAction< + Promise, + FetchAggStartedAction | FetchAggSuccessAction | FetchAggFailedAction +> { + return async function fetchAggregation( + dispatch, + getState, + { dataService, collection } + ) { + dispatch({ + type: VisualizationActionTypes.FETCH_AGG_STARTED, + }); + + try { + console.log('Running vector aggregation'); + const aggResults = await r( + `${collection.database}.${collection.name}`, + dataService, + 'funny', + 100, + 10 + ); + console.log( + 'Aggregation results:', + aggResults.candidates.slice(0, 10), + aggResults.limited.slice(0, 10) + ); + dispatch({ + type: VisualizationActionTypes.FETCH_AGG_SUCCESS, + aggResults, + }); + } catch (err) { + dispatch({ + type: VisualizationActionTypes.FETCH_AGG_FAILED, + error: err as Error, + }); + } + }; +} diff --git a/packages/compass-vector-embedding-visualizer/tsconfig.json b/packages/compass-vector-embedding-visualizer/tsconfig.json index 79bc84584ce..b56189e4574 100644 --- a/packages/compass-vector-embedding-visualizer/tsconfig.json +++ b/packages/compass-vector-embedding-visualizer/tsconfig.json @@ -1,7 +1,9 @@ { "extends": "@mongodb-js/tsconfig-compass/tsconfig.react.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "jsx": "react", + "lib": ["es2015", "dom"] }, "include": ["src/**/*"], "exclude": ["./src/**/*.spec.*"] diff --git a/packages/compass-workspaces/src/types.ts b/packages/compass-workspaces/src/types.ts index a744c060e61..d96ef77bab7 100644 --- a/packages/compass-workspaces/src/types.ts +++ b/packages/compass-workspaces/src/types.ts @@ -4,7 +4,8 @@ export type CollectionSubtab = | 'Schema' | 'Indexes' | 'Validation' - | 'GlobalWrites'; + | 'GlobalWrites' + | 'Vector Visualizer'; export type WelcomeWorkspace = { type: 'Welcome'; diff --git a/packages/compass/src/app/utils/csp.ts b/packages/compass/src/app/utils/csp.ts index bad57a89f13..3d4a62c0f1c 100644 --- a/packages/compass/src/app/utils/csp.ts +++ b/packages/compass/src/app/utils/csp.ts @@ -56,6 +56,7 @@ const defaultCSP = { 'https://cloud-qa.mongodb.com', 'https://compass.mongodb.com', 'https://ip-ranges.amazonaws.com', + 'https://api.voyageai.com', ], 'child-src': [ 'blob:',