diff --git a/migrations/tenant/0044-vector-bucket-type.sql b/migrations/tenant/0044-vector-bucket-type.sql new file mode 100644 index 00000000..6ba4ec58 --- /dev/null +++ b/migrations/tenant/0044-vector-bucket-type.sql @@ -0,0 +1,13 @@ +DO $$ + DECLARE + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_enum + JOIN pg_type ON pg_enum.enumtypid = pg_type.oid + WHERE pg_type.typname = 'buckettype' + AND enumlabel = 'VECTOR' + ) THEN + ALTER TYPE storage.BucketType ADD VALUE 'VECTOR'; + END IF; +END$$; \ No newline at end of file diff --git a/migrations/tenant/0045-vector-buckets.sql b/migrations/tenant/0045-vector-buckets.sql new file mode 100644 index 00000000..23152dfd --- /dev/null +++ b/migrations/tenant/0045-vector-buckets.sql @@ -0,0 +1,34 @@ +DO $$ + DECLARE + anon_role text = COALESCE(current_setting('storage.anon_role', true), 'anon'); + authenticated_role text = COALESCE(current_setting('storage.authenticated_role', true), 'authenticated'); + service_role text = COALESCE(current_setting('storage.service_role', true), 'service_role'); + BEGIN + CREATE TABLE IF NOT EXISTS storage.buckets_vectors ( + id text not null primary key, + type storage.BucketType NOT NULL default 'VECTOR', + created_at timestamptz NOT NULL default now(), + updated_at timestamptz NOT NULL default now() + ); + + CREATE TABLE IF NOT EXISTS storage.vector_indexes + ( + id text primary key default gen_random_uuid(), + name text COLLATE "C" NOT NULL, + bucket_id text NOT NULL references storage.buckets_vectors (id), + data_type text NOT NULL, + dimension integer NOT NULL, + distance_metric text NOT NULL, + metadata_configuration jsonb NULL, + created_at timestamptz NOT NULL default now(), + updated_at timestamptz NOT NULL default now() + ); + + ALTER TABLE storage.buckets_vectors ENABLE ROW LEVEL SECURITY; + ALTER TABLE storage.vector_indexes ENABLE ROW LEVEL SECURITY; + + EXECUTE 'GRANT SELECT ON TABLE storage.buckets_vectors TO ' || service_role || ', ' || authenticated_role || ', ' || anon_role; + EXECUTE 'GRANT SELECT ON TABLE storage.vector_indexes TO ' || service_role || ', ' || authenticated_role || ', ' || anon_role; + + CREATE UNIQUE INDEX IF NOT EXISTS vector_indexes_name_bucket_id_idx ON storage.vector_indexes (name, bucket_id); +END$$; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 086a0eb8..67997c10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.795.0", "@aws-sdk/client-s3": "3.654.0", + "@aws-sdk/client-s3vectors": "^3.883.0", "@aws-sdk/lib-storage": "3.654.0", "@aws-sdk/s3-request-presigner": "3.654.0", "@fastify/accepts": "^4.3.0", @@ -75,6 +76,7 @@ "@babel/preset-typescript": "^7.27.0", "@types/async-retry": "^1.4.5", "@types/busboy": "^1.3.0", + "@types/cloneable-readable": "^2.0.3", "@types/crypto-js": "^4.1.1", "@types/fs-extra": "^9.0.13", "@types/glob": "^8.1.0", @@ -82,7 +84,7 @@ "@types/js-yaml": "^4.0.5", "@types/multistream": "^4.1.3", "@types/mustache": "^4.2.2", - "@types/node": "^20.11.5", + "@types/node": "^22.18.8", "@types/pg": "^8.6.4", "@types/stream-buffers": "^3.0.7", "@types/xml2js": "^0.4.14", @@ -1647,6 +1649,1029 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-s3vectors": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.883.0.tgz", + "integrity": "sha512-QtGyHqvih3it25HFnxudHg6J6FvlpSQs9aSN2IgpofAQGLENGVtKFtVlFay2KOlUFPh5aL1OOl5ONR70O8q/Eg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-node": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/client-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.883.0.tgz", + "integrity": "sha512-Ybjw76yPceEBO7+VLjy5+/Gr0A1UNymSDHda5w8tfsS2iHZt/vuD6wrYpHdLoUx4H5la8ZhwcSfK/+kmE+QLPw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/core": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.883.0.tgz", + "integrity": "sha512-FmkqnqBLkXi4YsBPbF6vzPa0m4XKUuvgKDbamfw4DZX2CzfBZH6UU4IwmjNV3ZM38m0xraHarK8KIbGSadN3wg==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@aws-sdk/xml-builder": "3.873.0", + "@smithy/core": "^3.9.2", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/signature-v4": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-utf8": "^4.0.0", + "fast-xml-parser": "5.2.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.883.0.tgz", + "integrity": "sha512-Z6tPBXPCodfhIF1rvQKoeRGMkwL6TK0xdl1UoMIA1x4AfBpPICAF77JkFBExk/pdiFYq1d04Qzddd/IiujSlLg==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.883.0.tgz", + "integrity": "sha512-P589ug1lMOOEYLTaQJjSP+Gee34za8Kk2LfteNQfO9SpByHFgGj++Sg8VyIe30eZL8Q+i4qTt24WDCz1c+dgYg==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-stream": "^4.2.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.883.0.tgz", + "integrity": "sha512-n6z9HTzuDEdugXvPiE/95VJXbF4/gBffdV/SRHDJKtDHaRuvp/gggbfmfVSTFouGVnlKPb2pQWQsW3Nr/Y3Lrw==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.883.0.tgz", + "integrity": "sha512-QIUhsatsrwfB9ZsKpmi0EySSfexVP61wgN7hr493DOileh2QsKW4XATEfsWNmx0dj9323Vg1Mix7bXtRfl9cGg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-ini": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.883.0.tgz", + "integrity": "sha512-m1shbHY/Vppy4EdddG9r8x64TO/9FsCjokp5HbKcZvVoTOTgUJrdT8q2TAQJ89+zYIJDqsKbqfrmfwJ1zOdnGQ==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.883.0.tgz", + "integrity": "sha512-37ve9Tult08HLXrJFHJM/sGB/vO7wzI6v1RUUfeTiShqx8ZQ5fTzCTNY/duO96jCtCexmFNSycpQzh7lDIf0aA==", + "dependencies": { + "@aws-sdk/client-sso": "3.883.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/token-providers": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.883.0.tgz", + "integrity": "sha512-SL82K9Jb0vpuTadqTO4Fpdu7SKtebZ3Yo4LZvk/U0UauVMlJj5ZTos0mFx1QSMB9/4TpqifYrSZcdnxgYg8Eqw==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.873.0.tgz", + "integrity": "sha512-KZ/W1uruWtMOs7D5j3KquOxzCnV79KQW9MjJFZM/M0l6KI8J6V3718MXxFHsTjUE4fpdV6SeCNLV1lwGygsjJA==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-logger": { + "version": "3.876.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.876.0.tgz", + "integrity": "sha512-cpWJhOuMSyz9oV25Z/CMHCBTgafDCbv7fHR80nlRrPdPZ8ETNsahwRgltXP1QJJ8r3X/c1kwpOR7tc+RabVzNA==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.873.0.tgz", + "integrity": "sha512-OtgY8EXOzRdEWR//WfPkA/fXl0+WwE8hq0y9iw2caNyKPtca85dzrrZWnPqyBK/cpImosrpR1iKMYr41XshsCg==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.883.0.tgz", + "integrity": "sha512-q58uLYnGLg7hsnWpdj7Cd1Ulsq1/PUJOHvAfgcBuiDE/+Fwh0DZxZZyjrU+Cr+dbeowIdUaOO8BEDDJ0CUenJw==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@smithy/core": "^3.9.2", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/nested-clients": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.883.0.tgz", + "integrity": "sha512-IhzDM+v0ga53GOOrZ9jmGNr7JU5OR6h6ZK9NgB7GXaa+gsDbqfUuXRwyKDYXldrTXf1sUR3vy1okWDXA7S2ejQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.873.0.tgz", + "integrity": "sha512-q9sPoef+BBG6PJnc4x60vK/bfVwvRWsPgcoQyIra057S/QGjq5VkjvNk6H8xedf6vnKlXNBwq9BaANBXnldUJg==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "@smithy/util-config-provider": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/token-providers": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.883.0.tgz", + "integrity": "sha512-tcj/Z5paGn9esxhmmkEW7gt39uNoIRbXG1UwJrfKu4zcTr89h86PDiIE2nxUO3CMQf1KgncPpr5WouPGzkh/QQ==", + "dependencies": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/types": { + "version": "3.862.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", + "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "dependencies": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-endpoints": { + "version": "3.879.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.879.0.tgz", + "integrity": "sha512-aVAJwGecYoEmbEFju3127TyJDF9qJsKDUUTRMDuS8tGn+QiWQFnfInmbt+el9GU1gEJupNTXV+E3e74y51fb7A==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-endpoints": "^3.0.7", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.873.0.tgz", + "integrity": "sha512-AcRdbK6o19yehEcywI43blIBhOCSo6UgyWcuOJX5CFF8k39xm1ILCjQlRRjchLAxWrm0lU0Q7XV90RiMMFMZtA==", + "dependencies": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.883.0.tgz", + "integrity": "sha512-28cQZqC+wsKUHGpTBr+afoIdjS6IoEJkMqcZsmo2Ag8LzmTa6BUWQenFYB0/9BmDy4PZFPUn+uX+rJgWKB+jzA==", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@aws-sdk/xml-builder": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.873.0.tgz", + "integrity": "sha512-kLO7k7cGJ6KaHiExSJWojZurF7SnGMDHXRuQunFnEoD0n1yB6Lqy/S/zHiQ7oJnBhPr9q0TW9qFkrsZb1Uc54w==", + "dependencies": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/abort-controller": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.1.0.tgz", + "integrity": "sha512-wEhSYznxOmx7EdwK1tYEWJF5+/wmSFsff9BfTOn8oO/+KPl3gsmThrb6MJlWbOC391+Ya31s5JuHiC2RlT80Zg==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/config-resolver": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.2.0.tgz", + "integrity": "sha512-FA10YhPFLy23uxeWu7pOM2ctlw+gzbPMTZQwrZ8FRIfyJ/p8YIVz7AVTB5jjLD+QIerydyKcVMZur8qzzDILAQ==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-config-provider": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/core": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.10.0.tgz", + "integrity": "sha512-bXyD3Ij6b1qDymEYlEcF+QIjwb9gObwZNaRjETJsUEvSIzxFdynSQ3E4ysY7lUFSBzeWBNaFvX+5A0smbC2q6A==", + "dependencies": { + "@smithy/middleware-serde": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-body-length-browser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-stream": "^4.3.0", + "@smithy/util-utf8": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/credential-provider-imds": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.1.0.tgz", + "integrity": "sha512-iVwNhxTsCQTPdp++4C/d9xvaDmuEWhXi55qJobMp9QMaEHRGH3kErU4F8gohtdsawRqnUy/ANylCjKuhcR2mPw==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/fetch-http-handler": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.2.0.tgz", + "integrity": "sha512-VZenjDdVaUGiy3hwQtxm75nhXZrhFG+3xyL93qCQAlYDyhT/jeDWM8/3r5uCFMlTmmyrIjiDyiOynVFchb0BSg==", + "dependencies": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/hash-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.1.0.tgz", + "integrity": "sha512-mXkJQ/6lAXTuoSsEH+d/fHa4ms4qV5LqYoPLYhmhCRTNcMMdg+4Ya8cMgU1W8+OR40eX0kzsExT7fAILqtTl2w==", + "dependencies": { + "@smithy/types": "^4.4.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/invalid-dependency": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.1.0.tgz", + "integrity": "sha512-4/FcV6aCMzgpM4YyA/GRzTtG28G0RQJcWK722MmpIgzOyfSceWcI9T9c8matpHU9qYYLaWtk8pSGNCLn5kzDRw==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/is-array-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.1.0.tgz", + "integrity": "sha512-ePTYUOV54wMogio+he4pBybe8fwg4sDvEVDBU8ZlHOZXbXK3/C0XfJgUCu6qAZcawv05ZhZzODGUerFBPsPUDQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-content-length": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.1.0.tgz", + "integrity": "sha512-x3dgLFubk/ClKVniJu+ELeZGk4mq7Iv0HgCRUlxNUIcerHTLVmq7Q5eGJL0tOnUltY6KFw5YOKaYxwdcMwox/w==", + "dependencies": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-endpoint": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.2.0.tgz", + "integrity": "sha512-J1eCF7pPDwgv7fGwRd2+Y+H9hlIolF3OZ2PjptonzzyOXXGh/1KGJAHpEcY1EX+WLlclKu2yC5k+9jWXdUG4YQ==", + "dependencies": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-serde": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-retry": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.2.0.tgz", + "integrity": "sha512-raL5oWYf5ALl3jCJrajE8enKJEnV/2wZkKS6mb3ZRY2tg3nj66ssdWy5Ps8E6Yu8Wqh3Tt+Sb9LozjvwZupq+A==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/service-error-classification": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-retry": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.1.0.tgz", + "integrity": "sha512-CtLFYlHt7c2VcztyVRc+25JLV4aGpmaSv9F1sPB0AGFL6S+RPythkqpGDa2XBQLJQooKkjLA1g7Xe4450knShg==", + "dependencies": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/middleware-stack": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.1.0.tgz", + "integrity": "sha512-91Fuw4IKp0eK8PNhMXrHRcYA1jvbZ9BJGT91wwPy3bTQT8mHTcQNius/EhSQTlT9QUI3Ki1wjHeNXbWK0tO8YQ==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/node-config-provider": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.2.0.tgz", + "integrity": "sha512-8/fpilqKurQ+f8nFvoFkJ0lrymoMJ+5/CQV5IcTv/MyKhk2Q/EFYCAgTSWHD4nMi9ux9NyBBynkyE9SLg2uSLA==", + "dependencies": { + "@smithy/property-provider": "^4.1.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/node-http-handler": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.2.0.tgz", + "integrity": "sha512-G4NV70B4hF9vBrUkkvNfWO6+QR4jYjeO4tc+4XrKCb4nPYj49V9Hu8Ftio7Mb0/0IlFyEOORudHrm+isY29nCA==", + "dependencies": { + "@smithy/abort-controller": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/property-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.1.0.tgz", + "integrity": "sha512-eksMjMHUlG5PwOUWO3k+rfLNOPVPJ70mUzyYNKb5lvyIuAwS4zpWGsxGiuT74DFWonW0xRNy+jgzGauUzX7SyA==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/protocol-http": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.2.0.tgz", + "integrity": "sha512-bwjlh5JwdOQnA01be+5UvHK4HQz4iaRKlVG46hHSJuqi0Ribt3K06Z1oQ29i35Np4G9MCDgkOGcHVyLMreMcbg==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/querystring-builder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.1.0.tgz", + "integrity": "sha512-JqTWmVIq4AF8R8OK/2cCCiQo5ZJ0SRPsDkDgLO5/3z8xxuUp1oMIBBjfuueEe+11hGTZ6rRebzYikpKc6yQV9Q==", + "dependencies": { + "@smithy/types": "^4.4.0", + "@smithy/util-uri-escape": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/querystring-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.1.0.tgz", + "integrity": "sha512-VgdHhr8YTRsjOl4hnKFm7xEMOCRTnKw3FJ1nU+dlWNhdt/7eEtxtkdrJdx7PlRTabdANTmvyjE4umUl9cK4awg==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/service-error-classification": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.1.0.tgz", + "integrity": "sha512-UBpNFzBNmS20jJomuYn++Y+soF8rOK9AvIGjS9yGP6uRXF5rP18h4FDUsoNpWTlSsmiJ87e2DpZo9ywzSMH7PQ==", + "dependencies": { + "@smithy/types": "^4.4.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/shared-ini-file-loader": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.1.0.tgz", + "integrity": "sha512-W0VMlz9yGdQ/0ZAgWICFjFHTVU0YSfGoCVpKaExRM/FDkTeP/yz8OKvjtGjs6oFokCRm0srgj/g4Cg0xuHu8Rw==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/signature-v4": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.2.0.tgz", + "integrity": "sha512-ObX1ZqG2DdZQlXx9mLD7yAR8AGb7yXurGm+iWx9x4l1fBZ8CZN2BRT09aSbcXVPZXWGdn5VtMuupjxhOTI2EjA==", + "dependencies": { + "@smithy/is-array-buffer": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-uri-escape": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/smithy-client": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.6.0.tgz", + "integrity": "sha512-TvlIshqx5PIi0I0AiR+PluCpJ8olVG++xbYkAIGCUkByaMUlfOXLgjQTmYbr46k4wuDe8eHiTIlUflnjK2drPQ==", + "dependencies": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-endpoint": "^4.2.0", + "@smithy/middleware-stack": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-stream": "^4.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/types": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.4.0.tgz", + "integrity": "sha512-4jY91NgZz+ZnSFcVzWwngOW6VuK3gR/ihTwSU1R/0NENe9Jd8SfWgbhDCAGUWL3bI7DiDSW7XF6Ui6bBBjrqXw==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/url-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.1.0.tgz", + "integrity": "sha512-/LYEIOuO5B2u++tKr1NxNxhZTrr3A63jW8N73YTwVeUyAlbB/YM+hkftsvtKAcMt3ADYo0FsF1GY3anehffSVQ==", + "dependencies": { + "@smithy/querystring-parser": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-base64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.1.0.tgz", + "integrity": "sha512-RUGd4wNb8GeW7xk+AY5ghGnIwM96V0l2uzvs/uVHf+tIuVX2WSvynk5CxNoBCsM2rQRSZElAo9rt3G5mJ/gktQ==", + "dependencies": { + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-body-length-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.1.0.tgz", + "integrity": "sha512-V2E2Iez+bo6bUMOTENPr6eEmepdY8Hbs+Uc1vkDKgKNA/brTJqOW/ai3JO1BGj9GbCeLqw90pbbH7HFQyFotGQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-body-length-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.1.0.tgz", + "integrity": "sha512-BOI5dYjheZdgR9XiEM3HJcEMCXSoqbzu7CzIgYrx0UtmvtC3tC2iDGpJLsSRFffUpy8ymsg2ARMP5fR8mtuUQQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-buffer-from": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.1.0.tgz", + "integrity": "sha512-N6yXcjfe/E+xKEccWEKzK6M+crMrlwaCepKja0pNnlSkm6SjAeLKKA++er5Ba0I17gvKfN/ThV+ZOx/CntKTVw==", + "dependencies": { + "@smithy/is-array-buffer": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-config-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.1.0.tgz", + "integrity": "sha512-swXz2vMjrP1ZusZWVTB/ai5gK+J8U0BWvP10v9fpcFvg+Xi/87LHvHfst2IgCs1i0v4qFZfGwCmeD/KNCdJZbQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-defaults-mode-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.1.0.tgz", + "integrity": "sha512-D27cLtJtC4EEeERJXS+JPoogz2tE5zeE3zhWSSu6ER5/wJ5gihUxIzoarDX6K1U27IFTHit5YfHqU4Y9RSGE0w==", + "dependencies": { + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-defaults-mode-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.1.0.tgz", + "integrity": "sha512-gnZo3u5dP1o87plKupg39alsbeIY1oFFnCyV2nI/++pL19vTtBLgOyftLEjPjuXmoKn2B2rskX8b7wtC/+3Okg==", + "dependencies": { + "@smithy/config-resolver": "^4.2.0", + "@smithy/credential-provider-imds": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-endpoints": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.1.0.tgz", + "integrity": "sha512-5LFg48KkunBVGrNs3dnQgLlMXJLVo7k9sdZV5su3rjO3c3DmQ2LwUZI0Zr49p89JWK6sB7KmzyI2fVcDsZkwuw==", + "dependencies": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-hex-encoding": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.1.0.tgz", + "integrity": "sha512-1LcueNN5GYC4tr8mo14yVYbh/Ur8jHhWOxniZXii+1+ePiIbsLZ5fEI0QQGtbRRP5mOhmooos+rLmVASGGoq5w==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-middleware": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.1.0.tgz", + "integrity": "sha512-612onNcKyxhP7/YOTKFTb2F6sPYtMRddlT5mZvYf1zduzaGzkYhpYIPxIeeEwBZFjnvEqe53Ijl2cYEfJ9d6/Q==", + "dependencies": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-retry": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.1.0.tgz", + "integrity": "sha512-5AGoBHb207xAKSVwaUnaER+L55WFY8o2RhlafELZR3mB0J91fpL+Qn+zgRkPzns3kccGaF2vy0HmNVBMWmN6dA==", + "dependencies": { + "@smithy/service-error-classification": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-stream": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.3.0.tgz", + "integrity": "sha512-ZOYS94jksDwvsCJtppHprUhsIscRnCKGr6FXCo3SxgQ31ECbza3wqDBqSy6IsAak+h/oAXb1+UYEBmDdseAjUQ==", + "dependencies": { + "@smithy/fetch-http-handler": "^5.2.0", + "@smithy/node-http-handler": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-uri-escape": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.1.0.tgz", + "integrity": "sha512-b0EFQkq35K5NHUYxU72JuoheM6+pytEVUGlTwiFxWFpmddA+Bpz3LgsPRIpBk8lnPE47yT7AF2Egc3jVnKLuPg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/@smithy/util-utf8": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.1.0.tgz", + "integrity": "sha512-mEu1/UIXAdNYuBcyEPbjScKi/+MQVXNIuY/7Cm5XLIWe319kDrT5SizBE95jqtmEXoDbGoZxKLCMttdZdqTZKQ==", + "dependencies": { + "@smithy/util-buffer-from": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/fast-xml-parser": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.5.tgz", + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@aws-sdk/client-s3vectors/node_modules/strnum": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.1.tgz", + "integrity": "sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ] + }, "node_modules/@aws-sdk/client-sso": { "version": "3.848.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.848.0.tgz", @@ -12118,21 +13143,6 @@ "ws": "^8.18.2" } }, - "node_modules/@kubernetes/client-node/node_modules/@types/node": { - "version": "22.18.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.6.tgz", - "integrity": "sha512-r8uszLPpeIWbNKtvWRt/DbVi5zbqZyj1PTmhRMqBMvDnaz1QpmSKujUtJLrqGZeoM8v72MfYggDceY4K1itzWQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@kubernetes/client-node/node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "license": "MIT" - }, "node_modules/@lukeed/ms": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", @@ -15892,6 +16902,15 @@ "@types/node": "*" } }, + "node_modules/@types/cloneable-readable": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/cloneable-readable/-/cloneable-readable-2.0.3.tgz", + "integrity": "sha512-+Ihof4L4iu9k4WTzYbJSkzUxt6f1wzXn6u48fZYxgST+BsC9bBHTOJ59Buy1/4sC9j7ZWF7bxDf/n/mrtk/nzw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.36", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", @@ -16017,11 +17036,11 @@ } }, "node_modules/@types/node": { - "version": "20.11.18", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.18.tgz", - "integrity": "sha512-ABT5VWnnYneSBcNWYSCuR05M826RoMyMSGiFivXGx6ZUIsXb9vn4643IEwkg2zbEOSgAiSogtapN2fgc4mAPlw==", + "version": "22.18.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.8.tgz", + "integrity": "sha512-pAZSHMiagDR7cARo/cch1f3rXy0AEXwsVsVH09FcyeJVAzCnGgmYis7P3JidtTUjyadhTeSo8TgRPswstghDaw==", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "node_modules/@types/node-fetch": { @@ -22836,9 +23855,9 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" }, "node_modules/unicode-canonical-property-names-ecmascript": { "version": "2.0.1", @@ -24460,6 +25479,828 @@ } } }, + "@aws-sdk/client-s3vectors": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3vectors/-/client-s3vectors-3.883.0.tgz", + "integrity": "sha512-QtGyHqvih3it25HFnxudHg6J6FvlpSQs9aSN2IgpofAQGLENGVtKFtVlFay2KOlUFPh5aL1OOl5ONR70O8q/Eg==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-node": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@aws-sdk/client-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.883.0.tgz", + "integrity": "sha512-Ybjw76yPceEBO7+VLjy5+/Gr0A1UNymSDHda5w8tfsS2iHZt/vuD6wrYpHdLoUx4H5la8ZhwcSfK/+kmE+QLPw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/core": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.883.0.tgz", + "integrity": "sha512-FmkqnqBLkXi4YsBPbF6vzPa0m4XKUuvgKDbamfw4DZX2CzfBZH6UU4IwmjNV3ZM38m0xraHarK8KIbGSadN3wg==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@aws-sdk/xml-builder": "3.873.0", + "@smithy/core": "^3.9.2", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/signature-v4": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-utf8": "^4.0.0", + "fast-xml-parser": "5.2.5", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-env": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.883.0.tgz", + "integrity": "sha512-Z6tPBXPCodfhIF1rvQKoeRGMkwL6TK0xdl1UoMIA1x4AfBpPICAF77JkFBExk/pdiFYq1d04Qzddd/IiujSlLg==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-http": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.883.0.tgz", + "integrity": "sha512-P589ug1lMOOEYLTaQJjSP+Gee34za8Kk2LfteNQfO9SpByHFgGj++Sg8VyIe30eZL8Q+i4qTt24WDCz1c+dgYg==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/property-provider": "^4.0.5", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/util-stream": "^4.2.4", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-ini": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.883.0.tgz", + "integrity": "sha512-n6z9HTzuDEdugXvPiE/95VJXbF4/gBffdV/SRHDJKtDHaRuvp/gggbfmfVSTFouGVnlKPb2pQWQsW3Nr/Y3Lrw==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.883.0.tgz", + "integrity": "sha512-QIUhsatsrwfB9ZsKpmi0EySSfexVP61wgN7hr493DOileh2QsKW4XATEfsWNmx0dj9323Vg1Mix7bXtRfl9cGg==", + "requires": { + "@aws-sdk/credential-provider-env": "3.883.0", + "@aws-sdk/credential-provider-http": "3.883.0", + "@aws-sdk/credential-provider-ini": "3.883.0", + "@aws-sdk/credential-provider-process": "3.883.0", + "@aws-sdk/credential-provider-sso": "3.883.0", + "@aws-sdk/credential-provider-web-identity": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/credential-provider-imds": "^4.0.7", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-process": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.883.0.tgz", + "integrity": "sha512-m1shbHY/Vppy4EdddG9r8x64TO/9FsCjokp5HbKcZvVoTOTgUJrdT8q2TAQJ89+zYIJDqsKbqfrmfwJ1zOdnGQ==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-sso": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.883.0.tgz", + "integrity": "sha512-37ve9Tult08HLXrJFHJM/sGB/vO7wzI6v1RUUfeTiShqx8ZQ5fTzCTNY/duO96jCtCexmFNSycpQzh7lDIf0aA==", + "requires": { + "@aws-sdk/client-sso": "3.883.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/token-providers": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-web-identity": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.883.0.tgz", + "integrity": "sha512-SL82K9Jb0vpuTadqTO4Fpdu7SKtebZ3Yo4LZvk/U0UauVMlJj5ZTos0mFx1QSMB9/4TpqifYrSZcdnxgYg8Eqw==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-host-header": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.873.0.tgz", + "integrity": "sha512-KZ/W1uruWtMOs7D5j3KquOxzCnV79KQW9MjJFZM/M0l6KI8J6V3718MXxFHsTjUE4fpdV6SeCNLV1lwGygsjJA==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-logger": { + "version": "3.876.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.876.0.tgz", + "integrity": "sha512-cpWJhOuMSyz9oV25Z/CMHCBTgafDCbv7fHR80nlRrPdPZ8ETNsahwRgltXP1QJJ8r3X/c1kwpOR7tc+RabVzNA==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-recursion-detection": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.873.0.tgz", + "integrity": "sha512-OtgY8EXOzRdEWR//WfPkA/fXl0+WwE8hq0y9iw2caNyKPtca85dzrrZWnPqyBK/cpImosrpR1iKMYr41XshsCg==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-user-agent": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.883.0.tgz", + "integrity": "sha512-q58uLYnGLg7hsnWpdj7Cd1Ulsq1/PUJOHvAfgcBuiDE/+Fwh0DZxZZyjrU+Cr+dbeowIdUaOO8BEDDJ0CUenJw==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@smithy/core": "^3.9.2", + "@smithy/protocol-http": "^5.1.3", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/nested-clients": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.883.0.tgz", + "integrity": "sha512-IhzDM+v0ga53GOOrZ9jmGNr7JU5OR6h6ZK9NgB7GXaa+gsDbqfUuXRwyKDYXldrTXf1sUR3vy1okWDXA7S2ejQ==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.883.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.876.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.879.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.883.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.9.2", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.21", + "@smithy/middleware-retry": "^4.1.22", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.5.2", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.29", + "@smithy/util-defaults-mode-node": "^4.0.29", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/region-config-resolver": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.873.0.tgz", + "integrity": "sha512-q9sPoef+BBG6PJnc4x60vK/bfVwvRWsPgcoQyIra057S/QGjq5VkjvNk6H8xedf6vnKlXNBwq9BaANBXnldUJg==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "@smithy/util-config-provider": "^4.0.0", + "@smithy/util-middleware": "^4.0.5", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/token-providers": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.883.0.tgz", + "integrity": "sha512-tcj/Z5paGn9esxhmmkEW7gt39uNoIRbXG1UwJrfKu4zcTr89h86PDiIE2nxUO3CMQf1KgncPpr5WouPGzkh/QQ==", + "requires": { + "@aws-sdk/core": "3.883.0", + "@aws-sdk/nested-clients": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/property-provider": "^4.0.5", + "@smithy/shared-ini-file-loader": "^4.0.5", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/types": { + "version": "3.862.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.862.0.tgz", + "integrity": "sha512-Bei+RL0cDxxV+lW2UezLbCYYNeJm6Nzee0TpW0FfyTRBhH9C1XQh4+x+IClriXvgBnRquTMMYsmJfvx8iyLKrg==", + "requires": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/util-endpoints": { + "version": "3.879.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.879.0.tgz", + "integrity": "sha512-aVAJwGecYoEmbEFju3127TyJDF9qJsKDUUTRMDuS8tGn+QiWQFnfInmbt+el9GU1gEJupNTXV+E3e74y51fb7A==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-endpoints": "^3.0.7", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/util-user-agent-browser": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.873.0.tgz", + "integrity": "sha512-AcRdbK6o19yehEcywI43blIBhOCSo6UgyWcuOJX5CFF8k39xm1ILCjQlRRjchLAxWrm0lU0Q7XV90RiMMFMZtA==", + "requires": { + "@aws-sdk/types": "3.862.0", + "@smithy/types": "^4.3.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/util-user-agent-node": { + "version": "3.883.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.883.0.tgz", + "integrity": "sha512-28cQZqC+wsKUHGpTBr+afoIdjS6IoEJkMqcZsmo2Ag8LzmTa6BUWQenFYB0/9BmDy4PZFPUn+uX+rJgWKB+jzA==", + "requires": { + "@aws-sdk/middleware-user-agent": "3.883.0", + "@aws-sdk/types": "3.862.0", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/xml-builder": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.873.0.tgz", + "integrity": "sha512-kLO7k7cGJ6KaHiExSJWojZurF7SnGMDHXRuQunFnEoD0n1yB6Lqy/S/zHiQ7oJnBhPr9q0TW9qFkrsZb1Uc54w==", + "requires": { + "@smithy/types": "^4.3.2", + "tslib": "^2.6.2" + } + }, + "@smithy/abort-controller": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.1.0.tgz", + "integrity": "sha512-wEhSYznxOmx7EdwK1tYEWJF5+/wmSFsff9BfTOn8oO/+KPl3gsmThrb6MJlWbOC391+Ya31s5JuHiC2RlT80Zg==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/config-resolver": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.2.0.tgz", + "integrity": "sha512-FA10YhPFLy23uxeWu7pOM2ctlw+gzbPMTZQwrZ8FRIfyJ/p8YIVz7AVTB5jjLD+QIerydyKcVMZur8qzzDILAQ==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-config-provider": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/core": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.10.0.tgz", + "integrity": "sha512-bXyD3Ij6b1qDymEYlEcF+QIjwb9gObwZNaRjETJsUEvSIzxFdynSQ3E4ysY7lUFSBzeWBNaFvX+5A0smbC2q6A==", + "requires": { + "@smithy/middleware-serde": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-body-length-browser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-stream": "^4.3.0", + "@smithy/util-utf8": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + } + }, + "@smithy/credential-provider-imds": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.1.0.tgz", + "integrity": "sha512-iVwNhxTsCQTPdp++4C/d9xvaDmuEWhXi55qJobMp9QMaEHRGH3kErU4F8gohtdsawRqnUy/ANylCjKuhcR2mPw==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/fetch-http-handler": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.2.0.tgz", + "integrity": "sha512-VZenjDdVaUGiy3hwQtxm75nhXZrhFG+3xyL93qCQAlYDyhT/jeDWM8/3r5uCFMlTmmyrIjiDyiOynVFchb0BSg==", + "requires": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/hash-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.1.0.tgz", + "integrity": "sha512-mXkJQ/6lAXTuoSsEH+d/fHa4ms4qV5LqYoPLYhmhCRTNcMMdg+4Ya8cMgU1W8+OR40eX0kzsExT7fAILqtTl2w==", + "requires": { + "@smithy/types": "^4.4.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/invalid-dependency": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.1.0.tgz", + "integrity": "sha512-4/FcV6aCMzgpM4YyA/GRzTtG28G0RQJcWK722MmpIgzOyfSceWcI9T9c8matpHU9qYYLaWtk8pSGNCLn5kzDRw==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/is-array-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.1.0.tgz", + "integrity": "sha512-ePTYUOV54wMogio+he4pBybe8fwg4sDvEVDBU8ZlHOZXbXK3/C0XfJgUCu6qAZcawv05ZhZzODGUerFBPsPUDQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-content-length": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.1.0.tgz", + "integrity": "sha512-x3dgLFubk/ClKVniJu+ELeZGk4mq7Iv0HgCRUlxNUIcerHTLVmq7Q5eGJL0tOnUltY6KFw5YOKaYxwdcMwox/w==", + "requires": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-endpoint": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.2.0.tgz", + "integrity": "sha512-J1eCF7pPDwgv7fGwRd2+Y+H9hlIolF3OZ2PjptonzzyOXXGh/1KGJAHpEcY1EX+WLlclKu2yC5k+9jWXdUG4YQ==", + "requires": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-serde": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "@smithy/url-parser": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-retry": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.2.0.tgz", + "integrity": "sha512-raL5oWYf5ALl3jCJrajE8enKJEnV/2wZkKS6mb3ZRY2tg3nj66ssdWy5Ps8E6Yu8Wqh3Tt+Sb9LozjvwZupq+A==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/service-error-classification": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-retry": "^4.1.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + } + }, + "@smithy/middleware-serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.1.0.tgz", + "integrity": "sha512-CtLFYlHt7c2VcztyVRc+25JLV4aGpmaSv9F1sPB0AGFL6S+RPythkqpGDa2XBQLJQooKkjLA1g7Xe4450knShg==", + "requires": { + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-stack": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.1.0.tgz", + "integrity": "sha512-91Fuw4IKp0eK8PNhMXrHRcYA1jvbZ9BJGT91wwPy3bTQT8mHTcQNius/EhSQTlT9QUI3Ki1wjHeNXbWK0tO8YQ==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-config-provider": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.2.0.tgz", + "integrity": "sha512-8/fpilqKurQ+f8nFvoFkJ0lrymoMJ+5/CQV5IcTv/MyKhk2Q/EFYCAgTSWHD4nMi9ux9NyBBynkyE9SLg2uSLA==", + "requires": { + "@smithy/property-provider": "^4.1.0", + "@smithy/shared-ini-file-loader": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-http-handler": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.2.0.tgz", + "integrity": "sha512-G4NV70B4hF9vBrUkkvNfWO6+QR4jYjeO4tc+4XrKCb4nPYj49V9Hu8Ftio7Mb0/0IlFyEOORudHrm+isY29nCA==", + "requires": { + "@smithy/abort-controller": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/querystring-builder": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/property-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.1.0.tgz", + "integrity": "sha512-eksMjMHUlG5PwOUWO3k+rfLNOPVPJ70mUzyYNKb5lvyIuAwS4zpWGsxGiuT74DFWonW0xRNy+jgzGauUzX7SyA==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/protocol-http": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.2.0.tgz", + "integrity": "sha512-bwjlh5JwdOQnA01be+5UvHK4HQz4iaRKlVG46hHSJuqi0Ribt3K06Z1oQ29i35Np4G9MCDgkOGcHVyLMreMcbg==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-builder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.1.0.tgz", + "integrity": "sha512-JqTWmVIq4AF8R8OK/2cCCiQo5ZJ0SRPsDkDgLO5/3z8xxuUp1oMIBBjfuueEe+11hGTZ6rRebzYikpKc6yQV9Q==", + "requires": { + "@smithy/types": "^4.4.0", + "@smithy/util-uri-escape": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.1.0.tgz", + "integrity": "sha512-VgdHhr8YTRsjOl4hnKFm7xEMOCRTnKw3FJ1nU+dlWNhdt/7eEtxtkdrJdx7PlRTabdANTmvyjE4umUl9cK4awg==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/service-error-classification": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.1.0.tgz", + "integrity": "sha512-UBpNFzBNmS20jJomuYn++Y+soF8rOK9AvIGjS9yGP6uRXF5rP18h4FDUsoNpWTlSsmiJ87e2DpZo9ywzSMH7PQ==", + "requires": { + "@smithy/types": "^4.4.0" + } + }, + "@smithy/shared-ini-file-loader": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.1.0.tgz", + "integrity": "sha512-W0VMlz9yGdQ/0ZAgWICFjFHTVU0YSfGoCVpKaExRM/FDkTeP/yz8OKvjtGjs6oFokCRm0srgj/g4Cg0xuHu8Rw==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/signature-v4": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.2.0.tgz", + "integrity": "sha512-ObX1ZqG2DdZQlXx9mLD7yAR8AGb7yXurGm+iWx9x4l1fBZ8CZN2BRT09aSbcXVPZXWGdn5VtMuupjxhOTI2EjA==", + "requires": { + "@smithy/is-array-buffer": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-middleware": "^4.1.0", + "@smithy/util-uri-escape": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/smithy-client": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.6.0.tgz", + "integrity": "sha512-TvlIshqx5PIi0I0AiR+PluCpJ8olVG++xbYkAIGCUkByaMUlfOXLgjQTmYbr46k4wuDe8eHiTIlUflnjK2drPQ==", + "requires": { + "@smithy/core": "^3.10.0", + "@smithy/middleware-endpoint": "^4.2.0", + "@smithy/middleware-stack": "^4.1.0", + "@smithy/protocol-http": "^5.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-stream": "^4.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/types": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.4.0.tgz", + "integrity": "sha512-4jY91NgZz+ZnSFcVzWwngOW6VuK3gR/ihTwSU1R/0NENe9Jd8SfWgbhDCAGUWL3bI7DiDSW7XF6Ui6bBBjrqXw==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/url-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.1.0.tgz", + "integrity": "sha512-/LYEIOuO5B2u++tKr1NxNxhZTrr3A63jW8N73YTwVeUyAlbB/YM+hkftsvtKAcMt3ADYo0FsF1GY3anehffSVQ==", + "requires": { + "@smithy/querystring-parser": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-base64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.1.0.tgz", + "integrity": "sha512-RUGd4wNb8GeW7xk+AY5ghGnIwM96V0l2uzvs/uVHf+tIuVX2WSvynk5CxNoBCsM2rQRSZElAo9rt3G5mJ/gktQ==", + "requires": { + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.1.0.tgz", + "integrity": "sha512-V2E2Iez+bo6bUMOTENPr6eEmepdY8Hbs+Uc1vkDKgKNA/brTJqOW/ai3JO1BGj9GbCeLqw90pbbH7HFQyFotGQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.1.0.tgz", + "integrity": "sha512-BOI5dYjheZdgR9XiEM3HJcEMCXSoqbzu7CzIgYrx0UtmvtC3tC2iDGpJLsSRFffUpy8ymsg2ARMP5fR8mtuUQQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.1.0.tgz", + "integrity": "sha512-N6yXcjfe/E+xKEccWEKzK6M+crMrlwaCepKja0pNnlSkm6SjAeLKKA++er5Ba0I17gvKfN/ThV+ZOx/CntKTVw==", + "requires": { + "@smithy/is-array-buffer": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-config-provider": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.1.0.tgz", + "integrity": "sha512-swXz2vMjrP1ZusZWVTB/ai5gK+J8U0BWvP10v9fpcFvg+Xi/87LHvHfst2IgCs1i0v4qFZfGwCmeD/KNCdJZbQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-browser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.1.0.tgz", + "integrity": "sha512-D27cLtJtC4EEeERJXS+JPoogz2tE5zeE3zhWSSu6ER5/wJ5gihUxIzoarDX6K1U27IFTHit5YfHqU4Y9RSGE0w==", + "requires": { + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-node": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.1.0.tgz", + "integrity": "sha512-gnZo3u5dP1o87plKupg39alsbeIY1oFFnCyV2nI/++pL19vTtBLgOyftLEjPjuXmoKn2B2rskX8b7wtC/+3Okg==", + "requires": { + "@smithy/config-resolver": "^4.2.0", + "@smithy/credential-provider-imds": "^4.1.0", + "@smithy/node-config-provider": "^4.2.0", + "@smithy/property-provider": "^4.1.0", + "@smithy/smithy-client": "^4.6.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-endpoints": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.1.0.tgz", + "integrity": "sha512-5LFg48KkunBVGrNs3dnQgLlMXJLVo7k9sdZV5su3rjO3c3DmQ2LwUZI0Zr49p89JWK6sB7KmzyI2fVcDsZkwuw==", + "requires": { + "@smithy/node-config-provider": "^4.2.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-hex-encoding": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.1.0.tgz", + "integrity": "sha512-1LcueNN5GYC4tr8mo14yVYbh/Ur8jHhWOxniZXii+1+ePiIbsLZ5fEI0QQGtbRRP5mOhmooos+rLmVASGGoq5w==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-middleware": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.1.0.tgz", + "integrity": "sha512-612onNcKyxhP7/YOTKFTb2F6sPYtMRddlT5mZvYf1zduzaGzkYhpYIPxIeeEwBZFjnvEqe53Ijl2cYEfJ9d6/Q==", + "requires": { + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-retry": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.1.0.tgz", + "integrity": "sha512-5AGoBHb207xAKSVwaUnaER+L55WFY8o2RhlafELZR3mB0J91fpL+Qn+zgRkPzns3kccGaF2vy0HmNVBMWmN6dA==", + "requires": { + "@smithy/service-error-classification": "^4.1.0", + "@smithy/types": "^4.4.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-stream": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.3.0.tgz", + "integrity": "sha512-ZOYS94jksDwvsCJtppHprUhsIscRnCKGr6FXCo3SxgQ31ECbza3wqDBqSy6IsAak+h/oAXb1+UYEBmDdseAjUQ==", + "requires": { + "@smithy/fetch-http-handler": "^5.2.0", + "@smithy/node-http-handler": "^4.2.0", + "@smithy/types": "^4.4.0", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-buffer-from": "^4.1.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-uri-escape": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.1.0.tgz", + "integrity": "sha512-b0EFQkq35K5NHUYxU72JuoheM6+pytEVUGlTwiFxWFpmddA+Bpz3LgsPRIpBk8lnPE47yT7AF2Egc3jVnKLuPg==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.1.0.tgz", + "integrity": "sha512-mEu1/UIXAdNYuBcyEPbjScKi/+MQVXNIuY/7Cm5XLIWe319kDrT5SizBE95jqtmEXoDbGoZxKLCMttdZdqTZKQ==", + "requires": { + "@smithy/util-buffer-from": "^4.1.0", + "tslib": "^2.6.2" + } + }, + "fast-xml-parser": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.5.tgz", + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "requires": { + "strnum": "^2.1.0" + } + }, + "strnum": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.1.tgz", + "integrity": "sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==" + } + } + }, "@aws-sdk/client-sso": { "version": "3.848.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.848.0.tgz", @@ -32218,21 +34059,6 @@ "stream-buffers": "^3.0.2", "tar-fs": "^3.0.8", "ws": "^8.18.2" - }, - "dependencies": { - "@types/node": { - "version": "22.18.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.6.tgz", - "integrity": "sha512-r8uszLPpeIWbNKtvWRt/DbVi5zbqZyj1PTmhRMqBMvDnaz1QpmSKujUtJLrqGZeoM8v72MfYggDceY4K1itzWQ==", - "requires": { - "undici-types": "~6.21.0" - } - }, - "undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" - } } }, "@lukeed/ms": { @@ -35098,6 +36924,15 @@ "@types/node": "*" } }, + "@types/cloneable-readable": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/cloneable-readable/-/cloneable-readable-2.0.3.tgz", + "integrity": "sha512-+Ihof4L4iu9k4WTzYbJSkzUxt6f1wzXn6u48fZYxgST+BsC9bBHTOJ59Buy1/4sC9j7ZWF7bxDf/n/mrtk/nzw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/connect": { "version": "3.4.36", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", @@ -35223,11 +37058,11 @@ } }, "@types/node": { - "version": "20.11.18", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.18.tgz", - "integrity": "sha512-ABT5VWnnYneSBcNWYSCuR05M826RoMyMSGiFivXGx6ZUIsXb9vn4643IEwkg2zbEOSgAiSogtapN2fgc4mAPlw==", + "version": "22.18.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.8.tgz", + "integrity": "sha512-pAZSHMiagDR7cARo/cch1f3rXy0AEXwsVsVH09FcyeJVAzCnGgmYis7P3JidtTUjyadhTeSo8TgRPswstghDaw==", "requires": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "@types/node-fetch": { @@ -40097,9 +41932,9 @@ "dev": true }, "undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" }, "unicode-canonical-property-names-ecmascript": { "version": "2.0.1", diff --git a/package.json b/package.json index efe5e5e1..d6146162 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.795.0", "@aws-sdk/client-s3": "3.654.0", + "@aws-sdk/client-s3vectors": "^3.883.0", "@aws-sdk/lib-storage": "3.654.0", "@aws-sdk/s3-request-presigner": "3.654.0", "@fastify/accepts": "^4.3.0", @@ -91,6 +92,7 @@ "@babel/preset-typescript": "^7.27.0", "@types/async-retry": "^1.4.5", "@types/busboy": "^1.3.0", + "@types/cloneable-readable": "^2.0.3", "@types/crypto-js": "^4.1.1", "@types/fs-extra": "^9.0.13", "@types/glob": "^8.1.0", @@ -98,7 +100,7 @@ "@types/js-yaml": "^4.0.5", "@types/multistream": "^4.1.3", "@types/mustache": "^4.2.2", - "@types/node": "^20.11.5", + "@types/node": "^22.18.8", "@types/pg": "^8.6.4", "@types/stream-buffers": "^3.0.7", "@types/xml2js": "^0.4.14", diff --git a/src/app.ts b/src/app.ts index dc99fe7e..472f8fde 100644 --- a/src/app.ts +++ b/src/app.ts @@ -63,6 +63,7 @@ const build = (opts: buildOpts = {}): FastifyInstance => { app.register(routes.cdn, { prefix: 'cdn' }) app.register(routes.healthcheck, { prefix: 'health' }) app.register(routes.iceberg, { prefix: 'iceberg/v1' }) + app.register(routes.vectors, { prefix: 'vectors' }) setErrorHandler(app) diff --git a/src/config.ts b/src/config.ts index c05d207c..2b773777 100644 --- a/src/config.ts +++ b/src/config.ts @@ -186,6 +186,9 @@ type StorageConfigType = { icebergBucketDetectionSuffix: string icebergBucketDetectionMode: 'BUCKET' | 'FULL_PATH' icebergS3DeleteEnabled: boolean + + vectorBucketS3?: string + vectorBucketRegion?: string } function getOptionalConfigFromEnv(key: string, fallback?: string): string | undefined { @@ -524,6 +527,9 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { ), icebergMaxTableCount: parseInt(getOptionalConfigFromEnv('ICEBERG_MAX_TABLES') || '10', 10), icebergS3DeleteEnabled: getOptionalConfigFromEnv('ICEBERG_S3_DELETE_ENABLED') === 'true', + + vectorBucketS3: getOptionalConfigFromEnv('VECTOR_BUCKET_S3') || undefined, + vectorBucketRegion: getOptionalConfigFromEnv('VECTOR_BUCKET_REGION') || undefined, } as StorageConfigType const serviceKey = getOptionalConfigFromEnv('SERVICE_KEY') || '' diff --git a/src/http/plugins/index.ts b/src/http/plugins/index.ts index d8e02b9b..2c613202 100644 --- a/src/http/plugins/index.ts +++ b/src/http/plugins/index.ts @@ -11,3 +11,4 @@ export * from './signature-v4' export * from './tracing' export * from './signals' export * from './iceberg' +export * from './vector' diff --git a/src/http/plugins/jwt.ts b/src/http/plugins/jwt.ts index 0ef11a40..16a2aa2a 100644 --- a/src/http/plugins/jwt.ts +++ b/src/http/plugins/jwt.ts @@ -21,6 +21,7 @@ declare module 'fastify' { interface JWTPluginOptions { enforceJwtRoles?: string[] + skipIfAlreadyAuthenticated?: boolean } const { jwtCachingEnabled } = getConfig() @@ -33,6 +34,10 @@ export const jwt = fastifyPlugin( fastify.decorateRequest('jwtPayload', undefined) fastify.addHook('preHandler', async (request) => { + if (opts.skipIfAlreadyAuthenticated && request.isAuthenticated && request.jwtPayload) { + return + } + request.jwt = (request.headers.authorization || '').replace(BEARER, '') if (!request.jwt && request.routeOptions.config.allowInvalidJwt) { diff --git a/src/http/plugins/signature-v4.ts b/src/http/plugins/signature-v4.ts index 6c7a7674..8cf008a6 100644 --- a/src/http/plugins/signature-v4.ts +++ b/src/http/plugins/signature-v4.ts @@ -1,7 +1,7 @@ import { FastifyInstance, FastifyRequest } from 'fastify' import fastifyPlugin from 'fastify-plugin' import { getJwtSecret, getTenantConfig, s3CredentialsManager } from '@internal/database' -import { ClientSignature, SignatureV4 } from '@storage/protocols/s3' +import { ClientSignature, SignatureV4, SignatureV4Service } from '@storage/protocols/s3' import { signJWT, verifyJWT } from '@internal/auth' import { ERRORS } from '@internal/errors' @@ -11,6 +11,12 @@ import { ChunkSignatureV4Parser, V4StreamingAlgorithm, } from '@storage/protocols/s3/signature-v4-stream' +// @ts-expect-error - no types for compose +import { compose, Readable } from 'stream' +import { HashSpillWritable } from '@internal/streams/hash-stream' +import { RequestByteCounterStream } from '@internal/streams' +import { ByteLimitTransformStream } from '@storage/protocols/s3/byte-limit-stream' +import { Writable } from 'node:stream' const { anonKeyAsync, @@ -30,95 +36,160 @@ type AWSRequest = FastifyRequest<{ Querystring: { 'X-Amz-Credential'?: string } declare module 'fastify' { interface FastifyRequest { - multiPartFileStream?: MultipartFile streamingSignatureV4?: ChunkSignatureV4Parser + multiPartFileStream?: MultipartFile + bodySha256: string } } +const JWT_SHAPE = + /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)?$/ + export const signatureV4 = fastifyPlugin( - async function (fastify: FastifyInstance) { - fastify.addHook('preHandler', async (request: AWSRequest) => { - const clientSignature = await extractSignature(request) - - const sessionToken = clientSignature.sessionToken - - const { - signature: signatureV4, - claims, - token, - } = await createServerSignature(request.tenantId, clientSignature) - - let storagePrefix = s3ProtocolPrefix - if ( - requestAllowXForwardedPrefix && - typeof request.headers['x-forwarded-prefix'] === 'string' - ) { - storagePrefix = request.headers['x-forwarded-prefix'] - } + async function ( + fastify: FastifyInstance, + opts: { service?: SignatureV4Service; allowBodyHash?: boolean; skipIfJwtToken?: boolean } + ) { + // Use preParsing when allowing to pre-calculate the sha256 of the body + if (opts.allowBodyHash) { + fastify.addHook('preParsing', async (request: AWSRequest, reply, bodyPayload) => { + if ( + opts.skipIfJwtToken && + request.headers.authorization?.replace('Bearer ', '')?.match(JWT_SHAPE) + ) { + return bodyPayload + } - const isVerified = signatureV4.verify(clientSignature, { - url: request.url, - body: request.body as string | ReadableStream | Buffer, - headers: request.headers as Record, - method: request.method, - query: request.query as Record, - prefix: storagePrefix, + return await authorizeRequestSignV4( + request, + bodyPayload as Readable, + SignatureV4Service.S3VECTORS, + opts.allowBodyHash + ) }) + } - if (!isVerified && !sessionToken) { - throw ERRORS.SignatureDoesNotMatch( - 'The request signature we calculated does not match the signature you provided. Check your key and signing method.' - ) - } + // Use preHandler when not allowing to pre-calculate the sha256 of the body + if (!opts.allowBodyHash) { + fastify.addHook('preHandler', async (request: AWSRequest) => { + await authorizeRequestSignV4(request, request.raw as Readable, SignatureV4Service.S3) + }) + } + }, + { name: 'auth-signature-v4' } +) - if (!isVerified && sessionToken) { - throw ERRORS.SignatureDoesNotMatch( - 'The request signature we calculated does not match the signature you provided, Check your credentials. ' + - 'The session token should be a valid JWT token' - ) - } +/** + * Authorize incoming request with Signature V4 + * + * @param request + * @param body + * @param service + * @param allowBodyHash + */ +async function authorizeRequestSignV4( + request: AWSRequest, + body: string | Buffer | Readable, + service: SignatureV4Service, + allowBodyHash = false +) { + const clientSignature = await extractSignature(request) + + const sessionToken = clientSignature.sessionToken + + const { + signature: signatureV4, + claims, + token, + } = await createServerSignature(request.tenantId, clientSignature, service, allowBodyHash) + + let storagePrefix = s3ProtocolPrefix + if (requestAllowXForwardedPrefix && typeof request.headers['x-forwarded-prefix'] === 'string') { + storagePrefix = request.headers['x-forwarded-prefix'] + } - const { secret: jwtSecret, jwks } = await getJwtSecret(request.tenantId) - - if (token) { - const payload = await verifyJWT(token, jwtSecret, jwks) - request.jwt = token - request.jwtPayload = payload - request.owner = payload.sub - - if (SignatureV4.isChunkedUpload(request.headers)) { - request.streamingSignatureV4 = createStreamingSignatureV4Parser({ - signatureV4, - streamAlgorithm: request.headers['x-amz-content-sha256'] as V4StreamingAlgorithm, - clientSignature, - trailers: request.headers['x-amz-trailer'] as string, - }) - } - return - } + let hashStreamComposer: (Writable & { digestHex: () => string }) | undefined + let byteHasherStream: + | (Writable & { + digestHex: () => string + toReadable: (opts: { autoCleanup: boolean }) => Readable + }) + | undefined - if (!claims) { - throw ERRORS.AccessDenied('Missing claims') - } + if (allowBodyHash) { + byteHasherStream = new HashSpillWritable({ + alg: 'sha256', + limitInMemoryBytes: 1024 * 1024 * 5, // 5MB + }) + hashStreamComposer = compose(new ByteLimitTransformStream(1024 * 1024 * 20), byteHasherStream) + hashStreamComposer!.digestHex = byteHasherStream.digestHex.bind(byteHasherStream) + } - const jwt = await signJWT(claims, jwtSecret, '5m') + const isVerified = await signatureV4.verify(clientSignature, { + url: request.url, + body: body, + headers: request.headers as Record, + method: request.method, + query: request.query as Record, + prefix: storagePrefix, + payloadHasher: hashStreamComposer, + }) - request.jwt = jwt - request.jwtPayload = claims - request.owner = claims.sub + if (!isVerified && !sessionToken) { + throw ERRORS.SignatureDoesNotMatch( + 'The request signature we calculated does not match the signature you provided. Check your key and signing method.' + ) + } - if (SignatureV4.isChunkedUpload(request.headers)) { - request.streamingSignatureV4 = createStreamingSignatureV4Parser({ - signatureV4, - streamAlgorithm: request.headers['x-amz-content-sha256'] as V4StreamingAlgorithm, - clientSignature, - trailers: request.headers['x-amz-trailer'] as string, - }) - } + if (!isVerified && sessionToken) { + throw ERRORS.SignatureDoesNotMatch( + 'The request signature we calculated does not match the signature you provided, Check your credentials. ' + + 'The session token should be a valid JWT token' + ) + } + + const wasBodyHashed = allowBodyHash && byteHasherStream && byteHasherStream.writableEnded + + const returnStream = wasBodyHashed + ? byteHasherStream!.toReadable({ autoCleanup: true }) + : (body as Readable) + + const { secret: jwtSecret, jwks } = await getJwtSecret(request.tenantId) + + if (!token) { + if (!claims) { + throw ERRORS.AccessDenied('Missing claims') + } + + const jwt = await signJWT(claims, jwtSecret, '5m') + + request.isAuthenticated = true + request.jwt = jwt + request.jwtPayload = claims + request.owner = claims.sub + } else { + const payload = await verifyJWT(token, jwtSecret, jwks) + request.isAuthenticated = true + request.jwt = token + request.jwtPayload = payload + request.owner = payload.sub + } + + if (SignatureV4.isChunkedUpload(request.headers)) { + request.streamingSignatureV4 = createStreamingSignatureV4Parser({ + signatureV4, + streamAlgorithm: request.headers['x-amz-content-sha256'] as V4StreamingAlgorithm, + clientSignature, + trailers: request.headers['x-amz-trailer'] as string, }) - }, - { name: 'auth-signature-v4' } -) + } + + if (wasBodyHashed) { + return compose(returnStream, new RequestByteCounterStream()) + } + + return returnStream +} async function extractSignature(req: AWSRequest) { if (typeof req.headers.authorization === 'string') { @@ -156,9 +227,13 @@ async function extractSignature(req: AWSRequest) { throw ERRORS.AccessDenied('Missing signature') } -async function createServerSignature(tenantId: string, clientSignature: ClientSignature) { +async function createServerSignature( + tenantId: string, + clientSignature: ClientSignature, + awsService = SignatureV4Service.S3, + allowBodyHash = false +) { const awsRegion = storageS3Region - const awsService = 's3' if (clientSignature?.sessionToken) { const tenantAnonKey = isMultitenant @@ -172,6 +247,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + allowBodyHashing: allowBodyHash, nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: tenantId, @@ -193,6 +269,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + allowBodyHashing: allowBodyHash, nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: credential.accessKey, @@ -214,6 +291,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi const signature = new SignatureV4({ enforceRegion: s3ProtocolEnforceRegion, allowForwardedHeader: s3ProtocolAllowForwardedHeader, + allowBodyHashing: allowBodyHash, nonCanonicalForwardedHost: s3ProtocolNonCanonicalHostHeader, credentials: { accessKey: s3ProtocolAccessKeyId, diff --git a/src/http/plugins/vector.ts b/src/http/plugins/vector.ts new file mode 100644 index 00000000..7fd319a6 --- /dev/null +++ b/src/http/plugins/vector.ts @@ -0,0 +1,37 @@ +import fastifyPlugin from 'fastify-plugin' +import { FastifyInstance } from 'fastify' +import { multitenantKnex } from '@internal/database' +import { + createS3VectorClient, + KnexVectorMetadataDB, + VectorStoreManager, + S3Vector, +} from '@storage/protocols/vector' +import { getConfig } from '../../config' +import { ERRORS } from '@internal/errors' + +declare module 'fastify' { + interface FastifyRequest { + s3Vector: VectorStoreManager + } +} + +const { vectorBucketS3 } = getConfig() + +const s3VectorClient = createS3VectorClient() +const s3VectorAdapter = new S3Vector(s3VectorClient) + +export const s3vector = fastifyPlugin(async function (fastify: FastifyInstance) { + fastify.addHook('preHandler', async (req) => { + if (!vectorBucketS3) { + throw ERRORS.FeatureNotEnabled('vector', 'Vector service not configured') + } + + const db = req.db.pool.acquire() + const store = new KnexVectorMetadataDB(db) + req.s3Vector = new VectorStoreManager(s3VectorAdapter, store, { + tenantId: req.tenantId, + vectorBucketName: vectorBucketS3, + }) + }) +}) diff --git a/src/http/routes/index.ts b/src/http/routes/index.ts index d8527091..083bb0e4 100644 --- a/src/http/routes/index.ts +++ b/src/http/routes/index.ts @@ -6,4 +6,5 @@ export { default as healthcheck } from './health' export { default as s3 } from './s3' export { default as iceberg } from './iceberg' export { default as cdn } from './cdn' +export { default as vectors } from './vector' export * from './admin' diff --git a/src/http/routes/operations.ts b/src/http/routes/operations.ts index e14571cf..150b58aa 100644 --- a/src/http/routes/operations.ts +++ b/src/http/routes/operations.ts @@ -80,4 +80,21 @@ export const ROUTE_OPERATIONS = { ICEBERG_CREATE_TABLE: 'storage.iceberg.table.create', ICEBERG_DROP_TABLE: 'storage.iceberg.table.drop', ICEBERG_COMMIT_TABLE: 'storage.iceberg.table.commit', + + // Vector + CREATE_VECTOR_BUCKET: 'storage.vector.bucket.create', + DELETE_VECTOR_BUCKET: 'storage.vector.bucket.delete', + LIST_VECTOR_BUCKETS: 'storage.vector.bucket.list', + GET_VECTOR_BUCKET: 'storage.vector.bucket.get', + + CREATE_VECTOR_INDEX: 'storage.vector.index.create', + DELETE_VECTOR_INDEX: 'storage.vector.index.delete', + LIST_VECTOR_INDEXES: 'storage.vector.index.list', + GET_VECTOR_INDEX: 'storage.vector.index.get', + + GET_VECTORS: 'storage.vector.vectors.get', + PUT_VECTORS: 'storage.vector.vectors.put', + LIST_VECTORS: 'storage.vector.vectors.list', + QUERY_VECTORS: 'storage.vector.vectors.query', + DELETE_VECTORS: 'storage.vector.vectors.delete', } diff --git a/src/http/routes/vector/create-bucket.ts b/src/http/routes/vector/create-bucket.ts new file mode 100644 index 00000000..bf22a0eb --- /dev/null +++ b/src/http/routes/vector/create-bucket.ts @@ -0,0 +1,45 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const createVectorBucket = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'Create a vector bucket', +} as const + +interface createVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof createVectorBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/CreateVectorBucket', + { + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_VECTOR_BUCKET }, + }, + schema: { + ...createVectorBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.createBucket(request.body.vectorBucketName) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/create-index.ts b/src/http/routes/vector/create-index.ts new file mode 100644 index 00000000..e98c87d0 --- /dev/null +++ b/src/http/routes/vector/create-index.ts @@ -0,0 +1,66 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const createVectorIndex = { + type: 'object', + body: { + type: 'object', + properties: { + dataType: { type: 'string', enum: ['float32'] }, + dimension: { type: 'number', minimum: 1, maximum: 4096 }, + distanceMetric: { type: 'string', enum: ['cosine', 'euclidean'] }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + metadataConfiguration: { + type: 'object', + required: ['nonFilterableMetadataKeys'], + properties: { + nonFilterableMetadataKeys: { + type: 'array', + items: { type: 'string' }, + }, + }, + }, + vectorBucketName: { type: 'string' }, + }, + required: ['dataType', 'dimension', 'distanceMetric', 'indexName', 'vectorBucketName'], + }, + summary: 'Create a vector index', +} as const + +interface createVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof createVectorIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/CreateIndex', + { + config: { + operation: { type: ROUTE_OPERATIONS.CREATE_VECTOR_INDEX }, + }, + schema: { + ...createVectorIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.createVectorIndex(request.body) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/delete-bucket.ts b/src/http/routes/vector/delete-bucket.ts new file mode 100644 index 00000000..22db7098 --- /dev/null +++ b/src/http/routes/vector/delete-bucket.ts @@ -0,0 +1,45 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteVectorBucket = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'Create a vector bucket', +} as const + +interface deleteVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof deleteVectorBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/DeleteVectorBucket', + { + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_VECTOR_BUCKET }, + }, + schema: { + ...deleteVectorBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.deleteBucket(request.body.vectorBucketName) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/delete-index.ts b/src/http/routes/vector/delete-index.ts new file mode 100644 index 00000000..a556b8f2 --- /dev/null +++ b/src/http/routes/vector/delete-index.ts @@ -0,0 +1,53 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteVectorIndex = { + type: 'object', + body: { + type: 'object', + properties: { + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + vectorBucketName: { type: 'string' }, + }, + required: ['indexName', 'vectorBucketName'], + }, + summary: 'Delete a vector index', +} as const + +interface deleteVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof deleteVectorIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/DeleteIndex', + { + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_VECTOR_INDEX }, + }, + schema: { + ...deleteVectorIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.deleteIndex(request.body) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/delete-vectors.ts b/src/http/routes/vector/delete-vectors.ts new file mode 100644 index 00000000..94bb6a68 --- /dev/null +++ b/src/http/routes/vector/delete-vectors.ts @@ -0,0 +1,50 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const deleteVector = { + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexName: { type: 'string' }, + keys: { type: 'array', items: { type: 'string' } }, + }, + required: ['vectorBucketName', 'indexName', 'keys'], + }, + summary: 'Delete vectors from an index', +} as const + +interface deleteVectorRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof deleteVector)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/DeleteVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.DELETE_VECTORS }, + }, + schema: { + ...deleteVector, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + await request.s3Vector.deleteVectors({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + keys: request.body.keys, + }) + + return response.send() + } + ) +} diff --git a/src/http/routes/vector/get-bucket.ts b/src/http/routes/vector/get-bucket.ts new file mode 100644 index 00000000..28e6b558 --- /dev/null +++ b/src/http/routes/vector/get-bucket.ts @@ -0,0 +1,45 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const getVectorBucket = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'Create a vector bucket', +} as const + +interface getVectorBucketRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof getVectorBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/GetVectorBucket', + { + config: { + operation: { type: ROUTE_OPERATIONS.GET_VECTOR_BUCKET }, + }, + schema: { + ...getVectorBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const bucketResult = await request.s3Vector.getBucket(request.body) + + return response.send(bucketResult) + } + ) +} diff --git a/src/http/routes/vector/get-index.ts b/src/http/routes/vector/get-index.ts new file mode 100644 index 00000000..bff41d1a --- /dev/null +++ b/src/http/routes/vector/get-index.ts @@ -0,0 +1,64 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const getVectorIndex = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + }, + required: ['vectorBucketName', 'indexName'], + }, + summary: 'Get a vector index', +} as const + +interface getVectorIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof getVectorIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/GetIndex', + { + config: { + operation: { type: ROUTE_OPERATIONS.GET_VECTOR_INDEX }, + }, + schema: { + ...getVectorIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.getIndex({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + }) + + return response.send({ + ...indexResult, + index: { + ...indexResult.index, + creationTime: indexResult.index?.creationTime + ? Math.floor(indexResult.index?.creationTime?.getTime() / 1000) + : undefined, + }, + }) + } + ) +} diff --git a/src/http/routes/vector/get-vectors.ts b/src/http/routes/vector/get-vectors.ts new file mode 100644 index 00000000..7a8691d9 --- /dev/null +++ b/src/http/routes/vector/get-vectors.ts @@ -0,0 +1,49 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const getVectors = { + type: 'object', + body: { + type: 'object', + properties: { + indexName: { type: 'string' }, + keys: { type: 'array', items: { type: 'string' } }, + returnData: { type: 'boolean', default: false }, + returnMetadata: { type: 'boolean', default: false }, + vectorBucketName: { type: 'string' }, + }, + required: ['indexName', 'keys', 'vectorBucketName'], + }, + summary: 'Returns vector attributes', +} as const + +interface getVectorsRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof getVectors)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/GetVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.GET_VECTORS }, + }, + schema: { + ...getVectors, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.getVectors(request.body) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/index.ts b/src/http/routes/vector/index.ts new file mode 100644 index 00000000..b1c1b54b --- /dev/null +++ b/src/http/routes/vector/index.ts @@ -0,0 +1,56 @@ +import { FastifyInstance } from 'fastify' +import { db, dbSuperUser, jwt, s3vector, signatureV4 } from '../../plugins' +import { getConfig } from '../../../config' + +import createVectorBucket from './create-bucket' +import deleteVectorBucket from './delete-bucket' +import listVectorBuckets from './list-buckets' +import getVectorBucket from './get-bucket' + +import createVectorIndex from './create-index' +import deleteVectorIndex from './delete-index' +import listIndexes from './list-indexes' +import getIndex from './get-index' + +import getVectors from './get-vectors' +import putVectors from './put-vectors' +import listVectors from './list-vectors' +import queryVectors from './query-vectors' +import deleteVectors from './delete-vectors' +import { SignatureV4Service } from '@storage/protocols/s3' + +const { dbServiceRole } = getConfig() + +export default async function routes(fastify: FastifyInstance) { + fastify.register(async function authenticated(fastify) { + fastify.register(signatureV4, { + service: SignatureV4Service.S3VECTORS, + allowBodyHash: true, + skipIfJwtToken: true, + }) + + fastify.register(jwt, { + enforceJwtRoles: [dbServiceRole], + skipIfAlreadyAuthenticated: true, + }) + + fastify.register(dbSuperUser) + fastify.register(s3vector) + + fastify.register(createVectorIndex) + fastify.register(deleteVectorIndex) + fastify.register(listIndexes) + fastify.register(getIndex) + + fastify.register(createVectorBucket) + fastify.register(deleteVectorBucket) + fastify.register(listVectorBuckets) + fastify.register(getVectorBucket) + + fastify.register(putVectors) + fastify.register(queryVectors) + fastify.register(deleteVectors) + fastify.register(listVectors) + fastify.register(getVectors) + }) +} diff --git a/src/http/routes/vector/list-buckets.ts b/src/http/routes/vector/list-buckets.ts new file mode 100644 index 00000000..d5256c6c --- /dev/null +++ b/src/http/routes/vector/list-buckets.ts @@ -0,0 +1,46 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const listBucket = { + type: 'object', + body: { + type: 'object', + properties: { + maxResults: { type: 'number', minimum: 1, maximum: 500, default: 500 }, + nextToken: { type: 'string' }, + prefix: { type: 'string' }, + }, + }, + summary: 'List vector buckets', +} as const + +interface listBucketRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof listBucket)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/ListVectorBuckets', + { + config: { + operation: { type: ROUTE_OPERATIONS.LIST_VECTOR_BUCKETS }, + }, + schema: { + ...listBucket, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const listBucketsResult = await request.s3Vector.listBuckets(request.body) + + return response.send(listBucketsResult) + } + ) +} diff --git a/src/http/routes/vector/list-indexes.ts b/src/http/routes/vector/list-indexes.ts new file mode 100644 index 00000000..33c488a0 --- /dev/null +++ b/src/http/routes/vector/list-indexes.ts @@ -0,0 +1,51 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const listIndex = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + maxResults: { type: 'number', minimum: 1, maximum: 500, default: 500 }, + nextToken: { type: 'string' }, + prefix: { type: 'string' }, + }, + required: ['vectorBucketName'], + }, + summary: 'List indexes in a vector bucket', +} as const + +interface listIndexRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof listIndex)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/ListIndexes', + { + config: { + operation: { type: ROUTE_OPERATIONS.LIST_VECTOR_INDEXES }, + }, + schema: { + ...listIndex, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.listIndexes({ + ...request.body, + vectorBucketName: request.body.vectorBucketName, + }) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/list-vectors.ts b/src/http/routes/vector/list-vectors.ts new file mode 100644 index 00000000..e7e90636 --- /dev/null +++ b/src/http/routes/vector/list-vectors.ts @@ -0,0 +1,60 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const listVectors = { + type: 'object', + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexArn: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + maxResults: { type: 'number', minimum: 1, maximum: 500 }, + nextToken: { type: 'string' }, + returnData: { type: 'boolean' }, + returnMetadata: { type: 'boolean' }, + segmentCount: { type: 'number', minimum: 1, maximum: 16 }, + segmentIndex: { type: 'number', minimum: 0, maximum: 15 }, + }, + required: ['vectorBucketName', 'indexName'], + }, + summary: 'List vectors in a vector index', +} as const + +interface listVectorsRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof listVectors)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/ListVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.LIST_VECTORS }, + }, + schema: { + ...listVectors, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.listVectors(request.body) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/put-vectors.ts b/src/http/routes/vector/put-vectors.ts new file mode 100644 index 00000000..786a69b2 --- /dev/null +++ b/src/http/routes/vector/put-vectors.ts @@ -0,0 +1,84 @@ +import { FastifyInstance } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' + +const putVector = { + body: { + type: 'object', + properties: { + vectorBucketName: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + description: + '3-63 chars, lowercase letters, numbers, hyphens, dots; must start/end with letter or number. Must be unique within the vector bucket.', + }, + vectors: { + type: 'array', + minItems: 1, + maxItems: 500, + items: { + type: 'object', + properties: { + data: { + type: 'object', + properties: { + float32: { type: 'array', items: { type: 'number' } }, + }, + required: ['float32'], + }, + metadata: { + type: 'object', + additionalProperties: { type: 'string' }, + }, + key: { type: 'string' }, + }, + required: ['data'], + }, + }, + }, + required: ['vectorBucketName', 'indexName', 'vectors'], + }, + summary: 'Put vectors into an index', +} as const + +interface putVectorRequest extends AuthenticatedRequest { + Body: FromSchema<(typeof putVector)['body']> +} + +export default async function routes(fastify: FastifyInstance) { + fastify.post( + '/PutVectors', + { + config: { + operation: { type: ROUTE_OPERATIONS.PUT_VECTORS }, + }, + schema: { + ...putVector, + tags: ['vector'], + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.putVectors({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + vectors: request.body.vectors.map((v) => { + return { + ...v, + key: v.key || undefined, + } + }), + }) + + return response.send(indexResult) + } + ) +} diff --git a/src/http/routes/vector/query-vectors.ts b/src/http/routes/vector/query-vectors.ts new file mode 100644 index 00000000..18c5f659 --- /dev/null +++ b/src/http/routes/vector/query-vectors.ts @@ -0,0 +1,159 @@ +import { FastifyInstance, FastifySchemaCompiler } from 'fastify' +import { AuthenticatedRequest } from '../../types' +import { FromSchema } from 'json-schema-to-ts' +import { ERRORS } from '@internal/errors' +import { ROUTE_OPERATIONS } from '../operations' +import Ajv from 'ajv' + +const defs = { + $id: 'https://schemas.example.com/defs.json', + $defs: { + Primitive: { + anyOf: [{ type: 'string' }, { type: 'number' }, { type: 'boolean' }], + }, + FieldOperators: { + type: 'object', + // ensure at least one operator remains after removal + minProperties: 1, + // only allow keys that start with '$' + propertyNames: { pattern: '^\\$' }, + properties: { + $eq: { $ref: '#/$defs/Primitive' }, + $ne: { $ref: '#/$defs/Primitive' }, + $gt: { type: 'number' }, + $gte: { type: 'number' }, + $lt: { type: 'number' }, + $lte: { type: 'number' }, + $in: { type: 'array', minItems: 1, items: { $ref: '#/$defs/Primitive' } }, + $nin: { type: 'array', minItems: 1, items: { $ref: '#/$defs/Primitive' } }, + $exists: { type: 'boolean' }, + }, + additionalProperties: false, + }, + LogicalFilter: { + anyOf: [ + { + type: 'object', + properties: { + $and: { + type: 'array', + minItems: 1, + items: { $ref: '#/$defs/Filter' }, + }, + }, + required: ['$and'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + $or: { + type: 'array', + minItems: 1, + items: { $ref: '#/$defs/Filter' }, + }, + }, + required: ['$or'], + additionalProperties: false, + }, + ], + }, + Filter: { + anyOf: [ + { $ref: '#/$defs/LogicalFilter' }, + { + type: 'object', + additionalProperties: { + anyOf: [{ $ref: '#/$defs/Primitive' }, { $ref: '#/$defs/FieldOperators' }], + }, + }, + ], + }, + }, +} as const + +const queryVectorBody = { + $id: 'https://schemas.example.com/queryVectorBody.json', + type: 'object', + properties: { + filter: { $ref: 'https://schemas.example.com/defs.json#/$defs/Filter' }, + indexArn: { type: 'string' }, + indexName: { + type: 'string', + minLength: 3, + maxLength: 45, + pattern: '^[a-z0-9](?:[a-z0-9.-]{1,61})?[a-z0-9]$', + }, + queryVector: { + type: 'object', + properties: { + float32: { type: 'array', items: { type: 'number' } }, + }, + required: ['float32'], + additionalProperties: false, + }, + returnDistance: { type: 'boolean' }, + returnMetadata: { type: 'boolean' }, + topK: { type: 'number' }, + vectorBucketName: { type: 'string' }, + }, + required: ['vectorBucketName', 'indexName', 'queryVector', 'topK'], + additionalProperties: false, +} as const + +interface queryVectorRequest extends AuthenticatedRequest { + Body: FromSchema +} + +export default async function routes(fastify: FastifyInstance) { + const ajvNoRemoval = new Ajv({ + allErrors: true, + removeAdditional: false, // <- key bit + coerceTypes: false, + }) + + const perRouteValidator: FastifySchemaCompiler = ({ schema }) => { + const validate = ajvNoRemoval.compile(schema as object) + return (data) => { + const ok = validate(data) + if (ok) return { value: data } + return { error: new Error(JSON.stringify(validate.errors)) } + } + } + + ajvNoRemoval.addSchema(defs) + ajvNoRemoval.addSchema(queryVectorBody) + + fastify.post( + '/QueryVectors', + { + validatorCompiler: perRouteValidator, + config: { + operation: { type: ROUTE_OPERATIONS.QUERY_VECTORS }, + }, + schema: { + body: { $ref: 'https://schemas.example.com/queryVectorBody.json' }, + tags: ['vector'], + summary: 'Query vectors', + }, + }, + async (request, response) => { + if (!request.s3Vector) { + throw ERRORS.FeatureNotEnabled('vectorStore', 'Vector service not configured') + } + + const indexResult = await request.s3Vector.queryVectors({ + vectorBucketName: request.body.vectorBucketName, + indexName: request.body.indexName, + indexArn: request.body.indexArn, + queryVector: request.body.queryVector, + topK: request.body.topK, + filter: request.body.filter, + returnDistance: request.body.returnDistance, + returnMetadata: request.body.returnMetadata, + }) + + return response.send(indexResult) + } + ) +} diff --git a/src/internal/consistency/eventual.ts b/src/internal/consistency/eventual.ts new file mode 100644 index 00000000..f1cdaf1b --- /dev/null +++ b/src/internal/consistency/eventual.ts @@ -0,0 +1,75 @@ +import { BasePayload, Event } from '@internal/queue' +import { randomUUID } from 'crypto' +import { Job } from 'pg-boss' + +const callbacks = new Map Promise>() + +export function eventually(operation: string, params: P, fn: (params: P) => Promise) { + if (callbacks.has(operation)) { + throw new Error(`Operation ${operation} is already registered`) + } + callbacks.set(operation, fn) + + return async function (this: { tenantId: string }) { + try { + return await fn(params) + } catch (error) { + const opId = randomUUID() + const opName = `${operation}-${opId}` + + const job = { + opId, + operation: opName, + params, + $version: 'v1', + tenant: { ref: this.tenantId, host: '' }, + } + + AsyncInvoker.send(job) + } + } +} + +interface AsyncInvokerPayload

+ extends BasePayload { + opId: string + operation: string + params: P +} + +class AsyncInvoker extends Event { + protected static invokers = new Map Promise>() + static queueName = 'async-invoker' + + /** + * Registers a function to handle a specific operation and immediately executes it. + * + * @param job + * @param handler - Function that will handle this operation when invoked + * @returns Promise with the result of sending the operation to the queue + */ + static async registerAndExecute( + job: AsyncInvokerPayload, + handler: (params: TParams) => Promise + ): Promise { + // Register the handler function for this operation + this.registerHandler(job.operation, handler) + + // Execute the operation by sending the params + return await AsyncInvoker.send(job) + } + + /** + * Registers a handler function for a specific operation. + * + * @param operation - Unique identifier for the operation + * @param handler - Function that will handle this operation when invoked + */ + private static registerHandler( + operation: string, + handler: (params: TParams) => Promise + ): void { + this.invokers.set(operation, handler) + } + static async handle

(job: Job

) {} +} diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index deb27086..fd6292b8 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -40,5 +40,8 @@ export const DBMigration = { 'add-search-v2-sort-support': 39, 'fix-prefix-race-conditions-optimized': 40, 'add-object-level-update-trigger': 41, - 'fix-object-level': 42, + 'rollback-prefix-triggers': 42, + 'fix-object-level': 43, + 'vector-bucket-type': 44, + 'vector-buckets': 45, } diff --git a/src/internal/errors/codes.ts b/src/internal/errors/codes.ts index e1c63dd8..35475299 100644 --- a/src/internal/errors/codes.ts +++ b/src/internal/errors/codes.ts @@ -43,6 +43,10 @@ export enum ErrorCode { IcebergError = 'IcebergError', IcebergMaximumResourceLimit = 'IcebergMaximumResourceLimit', NoSuchCatalog = 'NoSuchCatalog', + + S3VectorConflictException = 'ConflictException', + S3VectorNotFoundException = 'NotFoundException', + S3VectorBucketNotEmpty = 'VectorBucketNotEmpty', } export const ERRORS = { @@ -421,6 +425,27 @@ export const ERRORS = { message: `Catalog name "${name}" not found`, }) }, + S3VectorConflictException(resource: string, name: string) { + return new StorageBackendError({ + code: ErrorCode.S3VectorConflictException, + httpStatusCode: 409, + message: `${resource} "${name}" already exists`, + }) + }, + S3VectorNotFoundException(resource: string, name: string) { + return new StorageBackendError({ + code: ErrorCode.S3VectorNotFoundException, + httpStatusCode: 404, + message: `resource "${name}" not found`, + }) + }, + S3VectorBucketNotEmpty(name: string) { + return new StorageBackendError({ + code: ErrorCode.S3VectorBucketNotEmpty, + httpStatusCode: 400, + message: `Vector Bucket "${name}" not empty`, + }) + }, } export function isStorageError(errorType: ErrorCode, error: any): error is StorageBackendError { diff --git a/src/internal/streams/byte-counter.ts b/src/internal/streams/byte-counter.ts index 0200c364..42455954 100644 --- a/src/internal/streams/byte-counter.ts +++ b/src/internal/streams/byte-counter.ts @@ -17,3 +17,13 @@ export const createByteCounterStream = () => { }, } } + +export class RequestByteCounterStream extends Transform { + public receivedEncodedLength = 0 + + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + this.receivedEncodedLength += chunk.length + + cb(null, chunk) + } +} diff --git a/src/internal/streams/hash-stream.ts b/src/internal/streams/hash-stream.ts new file mode 100644 index 00000000..16297592 --- /dev/null +++ b/src/internal/streams/hash-stream.ts @@ -0,0 +1,261 @@ +// HashSpillWritable.ts +import fs, { WriteStream } from 'node:fs' +import * as fsp from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { Readable, Writable, WritableOptions } from 'node:stream' +import { finished } from 'node:stream/promises' +import { createHash, randomUUID } from 'node:crypto' + +export interface HashSpillWritableOptions { + /** Max bytes to keep in memory before spilling to disk (required, > 0). */ + limitInMemoryBytes: number + /** Hash algorithm (default: 'sha256'). */ + alg?: string + /** Parent directory for temp dirs (default: os.tmpdir()). */ + tmpRoot?: string + /** Writable options to pass to base class (rarely needed). */ + writableOptions?: WritableOptions +} + +export interface ToReadableOptions { + /** + * If true and data spilled to disk, the spilled file/dir will be removed + * after the **last** reader closes/ends. + */ + autoCleanup?: boolean +} + +/** + * Writable that hashes all bytes and buffers in memory up to `limitBytes`. + * On first overflow, it spills to a unique temp file and appends subsequent data there. + * - Call `digestHex()` *after* 'finish' (e.g. after pipeline resolves). + * - Get a fresh readable with `toReadable({ autoCleanup })`. If multiple readers + * are created, cleanup is deferred until the last one finishes. + * - Call `cleanup()` to explicitly remove temp artifacts; it defers until readers close. + */ +export class HashSpillWritable extends Writable { + private readonly limitBytes: number + private readonly alg: string + private readonly tmpRoot: string + + // Hashing + private hash = createHash('sha256') + + // Memory buffer until first spill + private chunks: Buffer[] | null = [] + private memSize = 0 + + // Spill state + private spilled = false + private tmpDir: string | null = null + private filePath: string | null = null + private fileStream: WriteStream | null = null + private ensureFilePromise: Promise | null = null + + // Readers + cleanup + private activeReaders = 0 + private cleanupPending = false + private cleanupRunning: Promise | null = null + + // Bookkeeping + private totalBytes = 0 + private finishedFlag = false + private digestValue: string | null = null + + constructor(opts: HashSpillWritableOptions) { + super(opts?.writableOptions) + if (!(opts?.limitInMemoryBytes > 0)) throw new Error('limitBytes must be a positive number') + + this.limitBytes = opts.limitInMemoryBytes + this.alg = opts.alg ?? 'sha256' + this.tmpRoot = opts.tmpRoot ?? os.tmpdir() + + this.hash = createHash(this.alg) + + this.on('error', () => { + void this.cleanupAsync() + }) + this.on('close', () => { + if (this.fileStream && !this.fileStream.closed) { + try { + this.fileStream.destroy() + } catch {} + } + }) + } + + // Writable implementation + _write(chunk: Buffer, _enc: BufferEncoding, cb: (error?: Error | null) => void): void { + try { + this.hash.update(chunk) + this.totalBytes += chunk.length + + if (!this.spilled) { + if (this.memSize + chunk.length <= this.limitBytes) { + this.chunks!.push(chunk) + this.memSize += chunk.length + cb() + return + } + // Spill + this.spilled = true + this.spillToDiskAndWrite(chunk).then(() => cb(), cb) + } else { + this.writeToFile(chunk).then(() => cb(), cb) + } + } catch (err) { + cb(err as Error) + } + } + + _final(cb: (error?: Error | null) => void): void { + const finalize = async () => { + if (this.spilled && this.fileStream) { + await new Promise((resolve, reject) => { + this.fileStream!.end((err: Error) => (err ? reject(err) : resolve())) + }) + await finished(this.fileStream).catch(() => {}) + } + if (!this.finishedFlag) { + this.digestValue = this.hash.digest('hex') + this.finishedFlag = true + } + } + finalize().then(() => cb(), cb) + } + + // Public API + + digestHex(): string { + if (!this.finishedFlag || !this.digestValue) { + throw new Error('digestHex() called before stream finished') + } + return this.digestValue + } + + size(): number { + return this.totalBytes + } + + toReadable(opts: ToReadableOptions = {}): Readable { + const { autoCleanup = false } = opts + + if (this.spilled) { + if (!this.filePath) throw new Error('Internal error: spilled but no filePath') + + const rs = fs.createReadStream(this.filePath) + this.activeReaders++ + + const done = () => { + rs.removeListener('close', done) + rs.removeListener('end', done) + rs.removeListener('error', done) + + this.activeReaders = Math.max(0, this.activeReaders - 1) + + if (autoCleanup && this.activeReaders === 0) { + this.cleanupPending = true + void this.maybeCleanupSpill() + } + } + + rs.once('close', done) + rs.once('end', done) + rs.once('error', done) + + return rs + } + + // In-memory: nothing to clean up + const snapshot = this.chunks ?? [] + return Readable.from(snapshot) + } + + /** Explicit cleanup (deferred if readers are still active). */ + async cleanup(): Promise { + this.cleanupPending = true + await this.maybeCleanupSpill() + } + + // Internals + + private async ensureFile(): Promise { + if (this.ensureFilePromise) return this.ensureFilePromise + + this.ensureFilePromise = (async () => { + this.tmpDir = await fsp.mkdtemp(path.join(this.tmpRoot, 'hashspill-')) // unique directory + const name = `${Date.now()}-${randomUUID()}.bin` // unique filename + this.filePath = path.join(this.tmpDir, name) + this.fileStream = fs.createWriteStream(this.filePath, { flags: 'wx' }) // fail if exists + })() + + return this.ensureFilePromise + } + + private writeToFile(buf: Buffer): Promise { + return new Promise(async (resolve, reject) => { + try { + await this.ensureFile() + const ok = this.fileStream!.write(buf) + if (ok) return resolve() + this.fileStream!.once('drain', resolve) + } catch (e) { + reject(e) + } + }) + } + + private async spillToDiskAndWrite(nextChunk: Buffer): Promise { + await this.ensureFile() + + const prefix = Buffer.concat(this.chunks!, this.memSize) + // Free memory + this.chunks = null + this.memSize = 0 + + await this.writeToFile(prefix) + await this.writeToFile(nextChunk) + } + + private async maybeCleanupSpill(): Promise { + if (this.cleanupRunning) return this.cleanupRunning + + this.cleanupRunning = (async () => { + try { + if (!this.spilled) return + if (!this.cleanupPending) return + if (this.activeReaders > 0) return + + // Ensure file stream is closed + try { + if (this.fileStream && !this.fileStream.destroyed) { + this.fileStream.destroy() + } + } catch {} + + // Remove file and directory (best-effort) + try { + if (this.filePath) await fsp.rm(this.filePath, { force: true }) + } catch {} + try { + if (this.tmpDir) await fsp.rm(this.tmpDir, { force: true, recursive: true }) + } catch {} + + // Null out for GC + this.filePath = null + this.tmpDir = null + this.fileStream = null + } finally { + this.cleanupRunning = null + } + })() + + return this.cleanupRunning + } + + private async cleanupAsync(): Promise { + this.cleanupPending = true + await this.maybeCleanupSpill() + } +} diff --git a/src/internal/streams/index.ts b/src/internal/streams/index.ts index 4fc27fe1..dcff245a 100644 --- a/src/internal/streams/index.ts +++ b/src/internal/streams/index.ts @@ -1,3 +1,4 @@ export * from './stream-speed' export * from './byte-counter' +export * from './hash-stream' export * from './monitor' diff --git a/src/storage/protocols/iceberg/catalog/tenant-catalog.ts b/src/storage/protocols/iceberg/catalog/tenant-catalog.ts index fd9ae46d..4c5b2d48 100644 --- a/src/storage/protocols/iceberg/catalog/tenant-catalog.ts +++ b/src/storage/protocols/iceberg/catalog/tenant-catalog.ts @@ -156,6 +156,7 @@ export class TenantAwareRestCatalog extends RestCatalogClient { ...rest, namespace: namespaceName, }) + await store.createTable({ name: rest.name, bucketId: catalog.id, diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index 9e6d5396..6753b4e9 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -1,9 +1,19 @@ import crypto from 'crypto' import { ERRORS } from '@internal/errors' +import { Readable } from 'stream' +import { createHash } from 'node:crypto' +import { pipeline } from 'stream/promises' +import { Writable } from 'node:stream' + +export enum SignatureV4Service { + S3 = 's3', + S3VECTORS = 's3vectors', +} interface SignatureV4Options { enforceRegion: boolean allowForwardedHeader?: boolean + allowBodyHashing?: boolean nonCanonicalForwardedHost?: string credentials: Omit & { secretKey: string } } @@ -23,11 +33,12 @@ export interface ClientSignature { interface SignatureRequest { url: string - body?: string | ReadableStream | Buffer + body?: string | ReadableStream | Buffer | Readable headers: Record method: string query?: Record prefix?: string + payloadHasher?: Writable & { digestHex: () => string } } interface Credentials { @@ -86,12 +97,14 @@ export class SignatureV4 { public readonly serverCredentials: SignatureV4Options['credentials'] enforceRegion: boolean allowForwardedHeader?: boolean + allowBodyHashing?: boolean nonCanonicalForwardedHost?: string constructor(options: SignatureV4Options) { this.serverCredentials = options.credentials this.enforceRegion = options.enforceRegion this.allowForwardedHeader = options.allowForwardedHeader + this.allowBodyHashing = options.allowBodyHashing this.nonCanonicalForwardedHost = options.nonCanonicalForwardedHost } @@ -252,12 +265,12 @@ export class SignatureV4 { * @param clientSignature * @param request */ - verify(clientSignature: ClientSignature, request: SignatureRequest) { + async verify(clientSignature: ClientSignature, request: SignatureRequest) { if (typeof clientSignature.policy?.raw === 'string') { return this.verifyPostPolicySignature(clientSignature, clientSignature.policy.raw) } - const serverSignature = this.sign(clientSignature, request) + const serverSignature = await this.sign(clientSignature, request) return crypto.timingSafeEqual( Buffer.from(clientSignature.signature), Buffer.from(serverSignature.signature) @@ -333,7 +346,7 @@ export class SignatureV4 { * @param clientSignature * @param request */ - sign(clientSignature: ClientSignature, request: SignatureRequest) { + async sign(clientSignature: ClientSignature, request: SignatureRequest) { const serverCredentials = this.serverCredentials this.validateCredentials(clientSignature.credentials) @@ -344,11 +357,12 @@ export class SignatureV4 { } const selectedRegion = this.getSelectedRegion(clientSignature.credentials.region) - const canonicalRequest = this.constructCanonicalRequest( + const canonicalRequest = await this.constructCanonicalRequest( clientSignature, request, clientSignature.signedHeaders ) + const stringToSign = this.constructStringToSign( longDate, clientSignature.credentials.shortDate, @@ -356,6 +370,7 @@ export class SignatureV4 { serverCredentials.service, canonicalRequest ) + const signingKey = this.signingKey( serverCredentials.secretKey, clientSignature.credentials.shortDate, @@ -366,7 +381,7 @@ export class SignatureV4 { return { signature: this.hmac(signingKey, stringToSign).toString('hex'), canonicalRequest } } - protected getPayloadHash(clientSignature: ClientSignature, request: SignatureRequest) { + protected async getPayloadHash(clientSignature: ClientSignature, request: SignatureRequest) { const body = request.body // For presigned URLs and GET requests, use UNSIGNED-PAYLOAD @@ -392,11 +407,18 @@ export class SignatureV4 { .digest('hex') } + // If body is a ReadableStream, calculate the SHA256 hash of the stream + if (body instanceof Readable && this.allowBodyHashing && request.payloadHasher) { + return await pipeline(body, request.payloadHasher).then(() => { + return request.payloadHasher?.digestHex() + }) + } + // Default to UNSIGNED-PAYLOAD if body is not a string or ArrayBuffer return 'UNSIGNED-PAYLOAD' } - protected constructCanonicalRequest( + protected async constructCanonicalRequest( clientSignature: ClientSignature, request: SignatureRequest, signedHeaders: string[] @@ -407,7 +429,7 @@ export class SignatureV4 { const canonicalQueryString = this.constructCanonicalQueryString(request.query || {}) const canonicalHeaders = this.constructCanonicalHeaders(request, signedHeaders) const signedHeadersString = signedHeaders.sort().join(';') - const payloadHash = this.getPayloadHash(clientSignature, request) + const payloadHash = await this.getPayloadHash(clientSignature, request) return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeadersString}\n${payloadHash}` } @@ -543,6 +565,14 @@ export class SignatureV4 { return this.hmac(kService, 'aws4_request') } + protected async sha256OfRequest(req: Readable) { + const hash = createHash('sha256') + for await (const chunk of req) { + hash.update(chunk) + } + return hash.digest('hex') + } + protected hmac(key: string | Buffer, data: string): Buffer { return crypto.createHmac('sha256', key).update(data).digest() } diff --git a/src/storage/protocols/vector/adapter/s3-vector.ts b/src/storage/protocols/vector/adapter/s3-vector.ts new file mode 100644 index 00000000..b73b46e4 --- /dev/null +++ b/src/storage/protocols/vector/adapter/s3-vector.ts @@ -0,0 +1,86 @@ +import { + CreateIndexCommand, + CreateIndexCommandInput, + CreateIndexCommandOutput, + DeleteIndexCommand, + DeleteIndexCommandInput, + DeleteIndexCommandOutput, + DeleteVectorsCommand, + DeleteVectorsInput, + DeleteVectorsOutput, + GetVectorsCommand, + GetVectorsCommandInput, + GetVectorsCommandOutput, + ListVectorsCommand, + ListVectorsInput, + ListVectorsOutput, + PutVectorsCommand, + PutVectorsInput, + PutVectorsOutput, + QueryVectorsCommand, + QueryVectorsInput, + QueryVectorsOutput, + S3VectorsClient, +} from '@aws-sdk/client-s3vectors' +import { getConfig } from '../../../../config' + +export interface VectorStore { + createVectorIndex(command: CreateIndexCommandInput): Promise + deleteVectorIndex(param: DeleteIndexCommandInput): Promise + putVectors(command: PutVectorsInput): Promise + listVectors(command: ListVectorsInput): Promise + + queryVectors(queryInput: QueryVectorsInput): Promise + + deleteVectors(deleteVectorsInput: DeleteVectorsInput): Promise + + getVectors(getVectorsInput: GetVectorsCommandInput): Promise +} + +const { storageS3Region, vectorBucketRegion } = getConfig() + +export function createS3VectorClient() { + const s3VectorClient = new S3VectorsClient({ + region: vectorBucketRegion || storageS3Region, + }) + + return new S3VectorsClient(s3VectorClient) +} + +export class S3Vector implements VectorStore { + constructor(protected readonly s3VectorClient: S3VectorsClient) {} + + getVectors(getVectorsInput: GetVectorsCommandInput): Promise { + return this.s3VectorClient.send(new GetVectorsCommand(getVectorsInput)) + } + + deleteVectors(deleteVectorsInput: DeleteVectorsInput): Promise { + return this.s3VectorClient.send(new DeleteVectorsCommand(deleteVectorsInput)) + } + + queryVectors(queryInput: QueryVectorsInput): Promise { + return this.s3VectorClient.send(new QueryVectorsCommand(queryInput)) + } + + async listVectors(command: ListVectorsInput): Promise { + return this.s3VectorClient.send(new ListVectorsCommand(command)) + } + + putVectors(command: PutVectorsInput): Promise { + const input = new PutVectorsCommand(command) + + return this.s3VectorClient.send(input) + } + + deleteVectorIndex(param: DeleteIndexCommandInput): Promise { + const command = new DeleteIndexCommand(param) + + return this.s3VectorClient.send(command) + } + + createVectorIndex(command: CreateIndexCommandInput): Promise { + const createIndexCommand = new CreateIndexCommand(command) + + return this.s3VectorClient.send(createIndexCommand) + } +} diff --git a/src/storage/protocols/vector/index.ts b/src/storage/protocols/vector/index.ts new file mode 100644 index 00000000..b872215b --- /dev/null +++ b/src/storage/protocols/vector/index.ts @@ -0,0 +1,3 @@ +export * from './vector-store' +export * from './adapter/s3-vector' +export * from './knex' diff --git a/src/storage/protocols/vector/knex.ts b/src/storage/protocols/vector/knex.ts new file mode 100644 index 00000000..e2ae8b7e --- /dev/null +++ b/src/storage/protocols/vector/knex.ts @@ -0,0 +1,208 @@ +import { Knex } from 'knex' +import { VectorIndex } from '@storage/schemas/vector' +import { ERRORS } from '@internal/errors' +import { VectorBucket } from '@storage/schemas' +import { ListVectorBucketsInput } from '@aws-sdk/client-s3vectors' +import { DatabaseError } from 'pg' + +type DBVectorIndex = VectorIndex & { id: string; created_at: Date; updated_at: Date } + +interface CreateVectorIndexParams { + dataType: string + dimension: number + distanceMetric: string + indexName: string + metadataConfiguration?: { + nonFilterableMetadataKeys?: string[] + } + vectorBucketName: string +} + +export interface ListIndexesInput { + bucketId: string + maxResults?: number + nextToken?: string | undefined + prefix?: string | undefined +} + +export interface ListIndexesResult { + indexes: Pick[] + nextToken?: string +} + +export interface ListBucketResult { + vectorBuckets: VectorBucket[] + nextToken?: string +} + +export interface VectorMetadataDB { + createVectorIndex(data: CreateVectorIndexParams): Promise + withTransaction( + fn: (db: KnexVectorMetadataDB) => T, + config?: Knex.TransactionConfig + ): Promise + deleteVectorIndex(bucketName: string, vectorIndexName: string): Promise + deleteVectorBucket(bucketName: string, vectorIndexName: string): Promise + getIndex(bucketId: string, name: string): Promise + + listIndexes(command: ListIndexesInput): Promise + + createVectorBucket(bucketName: string): Promise + + findVectorBucket(vectorBucketName: string): Promise + + findVectorIndexForBucket(vectorBucketName: string, indexName: string): Promise + + listBuckets(param: ListVectorBucketsInput): Promise +} + +export class KnexVectorMetadataDB implements VectorMetadataDB { + constructor(protected readonly knex: Knex) {} + + async listBuckets(param: ListVectorBucketsInput): Promise { + const query = this.knex.withSchema('storage').table('buckets_vectors') + if (param.prefix) { + query.where('id', 'like', `${param.prefix}%`) + } + + if (param.nextToken) { + query.andWhere('id', '>', param.nextToken) + } + const maxResults = param.maxResults ? Math.min(param.maxResults, 500) : 500 + + const result = await query.orderBy('id', 'asc').limit(maxResults + 1) + + const hasMore = result.length > maxResults + + const buckets = result.slice(0, maxResults) + + return { + vectorBuckets: buckets, + nextToken: hasMore ? buckets[buckets.length - 1].id : undefined, + } + } + + async findVectorIndexForBucket( + vectorBucketName: string, + indexName: string + ): Promise { + const index = await this.knex + .withSchema('storage') + .select('*') + .table('vector_indexes') + .where({ bucket_id: vectorBucketName, name: indexName }) + .first() + + if (!index) { + throw ERRORS.S3VectorNotFoundException('vector index', indexName) + } + return index + } + + async findVectorBucket(vectorBucketName: string): Promise { + const bucket = await this.knex + .withSchema('storage') + .table('buckets_vectors') + .where({ id: vectorBucketName }) + .first() + + if (!bucket) { + throw ERRORS.S3VectorNotFoundException('vector bucket', vectorBucketName) + } + + return bucket + } + + async createVectorBucket(bucketName: string): Promise { + try { + await this.knex.withSchema('storage').table('buckets_vectors').insert({ + id: bucketName, + }) + } catch (e) { + if (e instanceof Error && e instanceof DatabaseError) { + if (e.code === '23505') { + throw ERRORS.S3VectorConflictException('vector bucket', bucketName) + } + } + + throw e + } + } + + async listIndexes(command: ListIndexesInput): Promise { + const maxResults = command.maxResults ? Math.min(command.maxResults, 500) : 500 + + const query = this.knex + .withSchema('storage') + .select('name', 'bucket_id', 'created_at') + .from('vector_indexes') + .where({ bucket_id: command.bucketId }) + .orderBy('name', 'asc') + .table('vector_indexes') + + if (command.prefix) { + query.andWhere('name', 'like', `${command.prefix}%`) + } + + if (command.nextToken) { + query.andWhere('id', '>', command.nextToken) + } + + const result = await query.limit(maxResults + 1) + const hasMore = result.length > maxResults + + const indexes = result.slice(0, maxResults) + + return { + indexes, + nextToken: hasMore ? indexes[indexes.length - 1].name : undefined, + } + } + + async getIndex(bucketId: string, name: string): Promise { + const index = await this.knex + .withSchema('storage') + .select('*') + .table('vector_indexes') + .where({ bucket_id: bucketId, name: name }) + .first() + + if (!index) { + throw ERRORS.S3VectorNotFoundException('vector index', name) + } + return index + } + + createVectorIndex(data: CreateVectorIndexParams) { + return this.knex + .withSchema('storage') + .table('vector_indexes') + .insert({ + bucket_id: data.vectorBucketName, + data_type: data.dataType, + name: data.indexName, + dimension: data.dimension, + distance_metric: data.distanceMetric, + metadata_configuration: data.metadataConfiguration, + }) + } + + withTransaction(fn: (db: KnexVectorMetadataDB) => T): Promise { + return this.knex.transaction(async (trx) => { + const trxDb = new KnexVectorMetadataDB(trx) + return fn(trxDb) + }) + } + + deleteVectorIndex(bucketName: string, vectorIndexName: string): Promise { + return this.knex + .withSchema('storage') + .table('vector_indexes') + .where({ bucket_id: bucketName, name: vectorIndexName }) + .del() + } + + async deleteVectorBucket(bucketName: string) { + await this.knex.withSchema('storage').table('buckets_vectors').where({ id: bucketName }).del() + } +} diff --git a/src/storage/protocols/vector/vector-store.ts b/src/storage/protocols/vector/vector-store.ts new file mode 100644 index 00000000..b0a5fb5c --- /dev/null +++ b/src/storage/protocols/vector/vector-store.ts @@ -0,0 +1,290 @@ +import { + CreateIndexInput, + DeleteIndexInput, + DistanceMetric, + GetIndexCommandInput, + ListIndexesInput, + MetadataConfiguration, + GetIndexOutput, + PutVectorsInput, + ListVectorsInput, + ListVectorBucketsInput, + QueryVectorsInput, + DeleteVectorsInput, + GetVectorBucketInput, + GetVectorsCommandInput, +} from '@aws-sdk/client-s3vectors' +import { VectorMetadataDB } from './knex' +import { VectorStore } from './adapter/s3-vector' +import { ERRORS } from '@internal/errors' + +export class VectorStoreManager { + constructor( + protected readonly vectorStore: VectorStore, + protected readonly db: VectorMetadataDB, + protected readonly config: { tenantId: string; vectorBucketName: string } + ) {} + + protected getIndexName(name: string) { + return `${this.config.tenantId}-${name}` + } + + async createBucket(bucketName: string): Promise { + await this.db.createVectorBucket(bucketName) + } + + async deleteBucket(bucketName: string): Promise { + await this.db.withTransaction( + async (tx) => { + const indexes = await tx.listIndexes({ bucketId: bucketName, maxResults: 1 }) + + if (indexes.indexes.length > 0) { + throw ERRORS.S3VectorBucketNotEmpty(bucketName) + } + + await tx.deleteVectorBucket(bucketName) + }, + { isolationLevel: 'serializable' } + ) + } + + async getBucket(command: GetVectorBucketInput) { + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + const vectorBucket = await this.db.findVectorBucket(command.vectorBucketName) + + return { + vectorBucket: { + vectorBucketName: vectorBucket.id, + creationTime: Math.floor(vectorBucket.created_at?.getTime() / 1000), + }, + } + } + + async listBuckets(command: ListVectorBucketsInput) { + const bucketResult = await this.db.listBuckets({ + maxResults: command.maxResults, + nextToken: command.nextToken, + prefix: command.prefix, + }) + + return { + vectorBuckets: bucketResult.vectorBuckets.map((bucket) => ({ + vectorBucketName: bucket.id, + creationTime: Math.floor(bucket.created_at?.getTime() / 1000), + })), + nextToken: bucketResult.nextToken, + } + } + + async createVectorIndex(command: CreateIndexInput): Promise { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorBucket(command.vectorBucketName) + + const createIndexInput = { + ...command, + indexName: this.getIndexName(command.indexName), + } + + await this.db.withTransaction(async (tx) => { + await tx.createVectorIndex({ + dataType: createIndexInput.dataType!, + dimension: createIndexInput.dimension!, + distanceMetric: createIndexInput.distanceMetric!, + indexName: command.indexName!, + metadataConfiguration: createIndexInput.metadataConfiguration, + vectorBucketName: command.vectorBucketName!, + }) + + await this.vectorStore.createVectorIndex({ + ...createIndexInput, + vectorBucketName: this.config.vectorBucketName, + }) + }) + } + + async deleteIndex(command: DeleteIndexInput): Promise { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const vectorIndexName = this.getIndexName(command.indexName) + + await this.db.withTransaction(async (tx) => { + await tx.deleteVectorIndex(command.vectorBucketName!, vectorIndexName) + + await this.vectorStore.deleteVectorIndex({ + vectorBucketName: this.config.vectorBucketName, + indexName: vectorIndexName, + }) + }) + } + + async getIndex(command: GetIndexCommandInput): Promise { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + const index = await this.db.getIndex(command.vectorBucketName, command.indexName) + + return { + index: { + indexName: index.name, + dataType: index.data_type as 'float32', + dimension: index.dimension, + distanceMetric: index.distance_metric as DistanceMetric, + metadataConfiguration: index.metadata_configuration as MetadataConfiguration, + vectorBucketName: index.bucket_id, + creationTime: index.created_at, + indexArn: undefined, + }, + } + } + + async listIndexes(command: ListIndexesInput) { + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + const result = await this.db.listIndexes({ + bucketId: command.vectorBucketName, + maxResults: command.maxResults, + nextToken: command.nextToken, + prefix: command.prefix, + }) + + return { + indexes: result.indexes.map((i) => ({ + indexName: i.name, + vectorBucketName: i.bucket_id, + creationTime: Math.floor(i.created_at.getTime() / 1000), + })), + } + } + + async putVectors(command: PutVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const putVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + await this.vectorStore.putVectors(putVectorsInput) + } + + async deleteVectors(command: DeleteVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const deleteVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + + return this.vectorStore.deleteVectors(deleteVectorsInput) + } + + async listVectors(command: ListVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const listVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + + const result = await this.vectorStore.listVectors(listVectorsInput) + + return { + vectors: result.vectors, + nextToken: result.nextToken, + } + } + + async queryVectors(command: QueryVectorsInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const queryInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + return this.vectorStore.queryVectors(queryInput) + } + + async getVectors(command: GetVectorsCommandInput) { + if (!command.indexName) { + throw ERRORS.MissingParameter('indexName') + } + + if (!command.vectorBucketName) { + throw ERRORS.MissingParameter('vectorBucketName') + } + + await this.db.findVectorIndexForBucket(command.vectorBucketName, command.indexName) + + const getVectorsInput = { + ...command, + vectorBucketName: this.config.vectorBucketName, + indexName: this.getIndexName(command.indexName), + } + + const result = await this.vectorStore.getVectors(getVectorsInput) + + return { + vectors: result.vectors, + } + } +} diff --git a/src/storage/schemas/bucket.ts b/src/storage/schemas/bucket.ts index 08d16cf7..9ffd58d9 100644 --- a/src/storage/schemas/bucket.ts +++ b/src/storage/schemas/bucket.ts @@ -33,3 +33,4 @@ export const bucketSchema = { export type Bucket = FromSchema export type IcebergCatalog = Pick +export type VectorBucket = Pick & { created_at: Date } diff --git a/src/storage/schemas/vector.ts b/src/storage/schemas/vector.ts new file mode 100644 index 00000000..6ef23fd3 --- /dev/null +++ b/src/storage/schemas/vector.ts @@ -0,0 +1,25 @@ +import { FromSchema } from 'json-schema-to-ts' + +const vectorIndex = { + type: 'object', + properties: { + name: { type: 'string' }, + data_type: { type: 'string' }, + dimension: { type: 'number' }, + distance_metric: { type: 'string' }, + metadata_configuration: { + type: 'object', + properties: { + nonFilterableMetadataKeys: { + type: 'array', + items: { type: 'string' }, + }, + }, + }, + bucket_id: { type: 'string' }, + }, + required: ['name', 'dimension', 'distance_metric', 'bucket_id'], + additionalProperties: false, +} as const + +export type VectorIndex = FromSchema diff --git a/src/test/hash-stream.test.ts b/src/test/hash-stream.test.ts new file mode 100644 index 00000000..a10737d4 --- /dev/null +++ b/src/test/hash-stream.test.ts @@ -0,0 +1,598 @@ +import { Readable, Writable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import * as fsp from 'node:fs/promises' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createHash } from 'node:crypto' +import { HashSpillWritable } from '@internal/streams/hash-stream' + +function randBuf(size: number): Buffer { + const b = Buffer.allocUnsafe(size) + for (let i = 0; i < size; i++) b[i] = (i * 131 + 17) & 0xff // deterministic-ish + return b +} + +function readableFrom(...chunks: Buffer[]): Readable { + return Readable.from(chunks) +} + +async function dirEntries(p: string): Promise { + try { + const names = await fsp.readdir(p) + return names + } catch { + return [] + } +} + +async function countHashspillDirs(root: string): Promise { + const names = await dirEntries(root) + return names.filter((n) => n.startsWith('hashspill-')).length +} + +async function findSpillFilePath(root: string): Promise { + try { + const entries = await fsp.readdir(root, { withFileTypes: true }) + const dir = entries.find((e) => e.isDirectory() && e.name.startsWith('hashspill-')) + if (!dir) return null + const dirPath = path.join(root, dir.name) + const files = await fsp.readdir(dirPath) + if (files.length === 0) return null + return path.join(dirPath, files[0]) // our class writes a single file + } catch { + return null + } +} + +class SlowWritable extends Writable { + private delayMs: number + constructor(delayMs = 5) { + super({ highWaterMark: 16 * 1024 }) // small HWM to induce backpressure + this.delayMs = delayMs + } + _write(chunk: Buffer, _enc: BufferEncoding, cb: (e?: Error | null) => void) { + setTimeout(() => cb(), this.delayMs) + } +} + +describe('HashSpillWritable', () => { + let tmpRoot: string + + beforeEach(async () => { + tmpRoot = await fsp.mkdtemp(path.join(os.tmpdir(), 'hsw-tests-')) + }) + + afterEach(async () => { + // best-effort cleanup of left-overs + try { + await fsp.rm(tmpRoot, { recursive: true, force: true }) + } catch {} + }) + + test('in-memory: under limit stays in memory; digest & size are correct', async () => { + const limit = 1024 * 64 + const payload = randBuf(limit - 7) + const expectedDigest = createHash('sha256').update(payload).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + expect(sink.size()).toBe(payload.length) + expect(sink.digestHex()).toBe(expectedDigest) + + // toReadable returns in-memory stream + const collected: Buffer[] = [] + await pipeline( + sink.toReadable(), + new Writable({ + write(chunk, _enc, cb) { + collected.push(chunk as Buffer) + cb() + }, + }) + ) + expect(Buffer.concat(collected)).toEqual(payload) + + // No hashspill-* dirs should have been created + expect(await countHashspillDirs(tmpRoot)).toBe(0) + + // cleanup() should be a no-op + await expect(sink.cleanup()).resolves.toBeUndefined() + }) + + test('in-memory: exactly at limit does not spill', async () => { + const limit = 32 * 1024 + const payload = randBuf(limit) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(sink.size()).toBe(limit) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill: just over limit triggers spill; autoCleanup on reader removes artifacts', async () => { + const limit = 1024 * 32 + const payload = randBuf(limit + 1) + const expectedDigest = createHash('sha256').update(payload).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + expect(sink.digestHex()).toBe(expectedDigest) + // A spill should have created exactly one temp dir + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + // Read with autoCleanup so artifacts get removed when last reader ends + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Allow event loop to process cleanup + await new Promise((r) => setTimeout(r, 10)) + + // The hashspill dir should be gone + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill: multiple readers, autoCleanup waits for the last reader', async () => { + const limit = 8 * 1024 + const payload = randBuf(limit * 3) // force spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + const r1 = sink.toReadable({ autoCleanup: true }) + const r2 = sink.toReadable({ autoCleanup: true }) + + // pipe r1 quickly + const fastConsumer = new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + // r2 is slower + const slowConsumer = new SlowWritable(3) + + const p1 = pipeline(r1, fastConsumer) + const p2 = pipeline(r2, slowConsumer) + await Promise.all([p1, p2]) + + // wait a tick for cleanup to run + await new Promise((r) => setTimeout(r, 10)) + + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('manual cleanup: delete after readers close (call cleanup after reading)', async () => { + const limit = 4096 + const payload = randBuf(limit * 5) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + // No autoCleanup; we clean manually after + const r = sink.toReadable() + await pipeline( + r, + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Now manual cleanup removes artifacts + await sink.cleanup() + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('backpressure respected with slow downstream while hashing', async () => { + const limit = 16 * 1024 + const pieces = Array.from({ length: 50 }, (_, i) => randBuf(2048 + (i % 5))) + const payload = Buffer.concat(pieces) + const expectedDigest = createHash('sha256').update(payload).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + // Write into sink, then read out to a slow consumer to ensure stream semantics hold + await pipeline(Readable.from(pieces), sink) + const outPieces: Buffer[] = [] + await pipeline( + sink.toReadable(), + new SlowWritable(2).on('pipe', function () {}) + ) + + expect(sink.digestHex()).toBe(expectedDigest) + }) + + test('multiple concurrent instances (no collisions, all succeed)', async () => { + const N = 8 + const limit = 8 * 1024 + const jobs = Array.from({ length: N }, async (_, idx) => { + const buf = randBuf(limit + 1024 + idx) // force spill + const exp = createHash('sha256').update(buf).digest('hex') + + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(buf), sink) + + expect(sink.digestHex()).toBe(exp) + // Use autoCleanup to clean right after reading + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + }) + + await Promise.all(jobs) + + // Allow cleanup to finish + await new Promise((r) => setTimeout(r, 10)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('size() tracks total bytes written', async () => { + const limit = 10 * 1024 + const parts = [randBuf(1111), randBuf(2222), randBuf(3333)] + const total = parts.reduce((n, b) => n + b.length, 0) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(Readable.from(parts), sink) + expect(sink.size()).toBe(total) + }) + + test('toReadable() can be called multiple times (consistent replay)', async () => { + const limit = 4096 + const payload = randBuf(limit * 2) // spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + const readAll = async () => { + const chunks: Buffer[] = [] + await pipeline( + sink.toReadable(), + new Writable({ + write(c, _e, cb) { + chunks.push(c as Buffer) + cb() + }, + }) + ) + return Buffer.concat(chunks) + } + + const a = await readAll() + const b = await readAll() + expect(a).toEqual(payload) + expect(b).toEqual(payload) + + await sink.cleanup() + }) + + test('cleanup is a no-op for non-spilled streams', async () => { + const limit = 1 << 20 + const payload = randBuf(12345) // under limit + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + await expect(sink.cleanup()).resolves.toBeUndefined() + // Nothing created on disk + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('errors if digestHex() is called before finish', async () => { + const limit = 1024 + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + // start write but don't finish + const r = new Readable({ + read() { + this.push(randBuf(200)) + this.push(null) + }, + }) + await pipeline(r, sink) + // now finished — valid + expect(() => sink.digestHex()).not.toThrow() + }) + + test('spill: if file cannot be created/written, pipeline rejects with a handled error', async () => { + const limit = 8 * 1024 + const payload = randBuf(limit * 3) // force spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + // Stub createWriteStream to fail on creation + const spy = jest.spyOn(fs, 'createWriteStream').mockImplementation(() => { + throw Object.assign(new Error('simulated createWriteStream failure'), { code: 'EACCES' }) + }) + + try { + await expect(pipeline(readableFrom(payload), sink)).rejects.toThrow( + /createWriteStream failure|EACCES|simulated/i + ) + } finally { + spy.mockRestore() + } + + // Ensure no lingering temp dirs/files (best-effort) + await new Promise((r) => setTimeout(r, 10)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill: spilled file exists before read and is deleted after autoCleanup', async () => { + const limit = 16 * 1024 + const payload = randBuf(limit * 2 + 123) // force spill + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + + // The spill dir & file should exist now + const prePath = await findSpillFilePath(tmpRoot) + expect(prePath).not.toBeNull() + expect(fs.existsSync(prePath!)).toBe(true) + + // Read with autoCleanup + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Give the event loop a tick for cleanup + await new Promise((r) => setTimeout(r, 15)) + + // The specific spilled file AND directory should be gone + const postPath = await findSpillFilePath(tmpRoot) + expect(postPath).toBeNull() + expect(fs.existsSync(prePath!)).toBe(false) + + // And no hashspill dirs remain + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + test('concurrent spill operations: no temp file name collisions with rapid creation', async () => { + const limit = 4 * 1024 + const N = 20 // More instances to increase collision probability + + // Create all instances simultaneously to maximize collision chance + const sinks = Array.from( + { length: N }, + () => new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + ) + + // Start all writes concurrently + const writePromises = sinks.map(async (sink, idx) => { + const buf = randBuf(limit + 100 + idx) // force spill on all + await pipeline(readableFrom(buf), sink) + return { sink, expected: createHash('sha256').update(buf).digest('hex') } + }) + + const results = await Promise.all(writePromises) + + // Verify all succeeded with correct digests + for (const { sink, expected } of results) { + expect(sink.digestHex()).toBe(expected) + } + + // Verify all created separate temp directories + expect(await countHashspillDirs(tmpRoot)).toBe(N) + + // Clean up all with autoCleanup + const readPromises = results.map(({ sink }) => + pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + ) + + await Promise.all(readPromises) + await new Promise((r) => setTimeout(r, 20)) // Allow cleanup + + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('concurrent spill with identical timestamps: UUID ensures uniqueness', async () => { + const limit = 2 * 1024 + const N = 10 + + // Mock Date.now to return same timestamp for all instances + const originalDateNow = Date.now + const fixedTimestamp = 1234567890123 + jest.spyOn(Date, 'now').mockReturnValue(fixedTimestamp) + + try { + const jobs = Array.from({ length: N }, async (_, idx) => { + const payload = randBuf(limit + 50 + idx) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + // Verify file was created successfully despite same timestamp + const spillPath = await findSpillFilePath(tmpRoot) + expect(spillPath).not.toBeNull() + + await sink.cleanup() + return sink.digestHex() + }) + + // All should succeed despite identical timestamps + const digests = await Promise.all(jobs) + expect(digests).toHaveLength(N) + + // All temp dirs should be cleaned up + await new Promise((r) => setTimeout(r, 10)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + } finally { + jest.restoreAllMocks() + } + }) + + test('concurrent readers on same spilled stream with mixed cleanup strategies', async () => { + const limit = 8 * 1024 + const payload = randBuf(limit * 2) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + // Create multiple readers: some with autoCleanup, some without + const readers = [ + { stream: sink.toReadable({ autoCleanup: true }), name: 'auto1' }, + { stream: sink.toReadable({ autoCleanup: false }), name: 'manual1' }, + { stream: sink.toReadable({ autoCleanup: true }), name: 'auto2' }, + { stream: sink.toReadable({ autoCleanup: false }), name: 'manual2' }, + { stream: sink.toReadable({ autoCleanup: true }), name: 'auto3' }, + ] + + // Read from all concurrently with varying speeds + const readPromises = readers.map(({ stream, name }, idx) => { + const consumer = new SlowWritable(100 * idx + 1) // Different speeds + return pipeline(stream, consumer) + }) + + await Promise.all(readPromises) + + // Even though some had autoCleanup=true, cleanup should be deferred + // because other readers existed. Only manual cleanup should work now. + expect(await countHashspillDirs(tmpRoot)).toBe(1) + + await new Promise((r) => setTimeout(r, 200)) + // Manual cleanup should now succeed + await sink.cleanup() + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('rapid spill/cleanup cycles: no resource leaks or race conditions', async () => { + const limit = 4 * 1024 + const cycles = 15 + + for (let i = 0; i < cycles; i++) { + const payload = randBuf(limit + 100 + i) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + + await pipeline(readableFrom(payload), sink) + + // Immediately read and cleanup + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + // Brief pause to allow cleanup + await new Promise((r) => setTimeout(r, 2)) + } + + // All temp artifacts should be cleaned up + await new Promise((r) => setTimeout(r, 20)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) + + test('spill during concurrent writes to different tmp roots: isolation verified', async () => { + const limit = 6 * 1024 + const tmpRoot2 = await fsp.mkdtemp(path.join(os.tmpdir(), 'hsw-tests2-')) + + try { + const payload1 = randBuf(limit + 200) + const payload2 = randBuf(limit + 300) + + const sink1 = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + const sink2 = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot: tmpRoot2 }) + + // Write to both concurrently + await Promise.all([ + pipeline(readableFrom(payload1), sink1), + pipeline(readableFrom(payload2), sink2), + ]) + + // Each should have created temp dirs in their respective roots + expect(await countHashspillDirs(tmpRoot)).toBe(1) + expect(await countHashspillDirs(tmpRoot2)).toBe(1) + + // Cleanup both + await Promise.all([sink1.cleanup(), sink2.cleanup()]) + + expect(await countHashspillDirs(tmpRoot)).toBe(0) + expect(await countHashspillDirs(tmpRoot2)).toBe(0) + } finally { + await fsp.rm(tmpRoot2, { recursive: true, force: true }).catch(() => {}) + } + }) + + test('stress test: many concurrent spills with overlapping lifecycles', async () => { + const limit = 8 * 1024 + const batchSize = 12 + + // Create overlapping batches + const batch1Promise = Promise.all( + Array.from({ length: batchSize }, async (_, i) => { + const payload = randBuf(limit * 2 + i) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + // Delay before reading to create overlap with batch2 + await new Promise((r) => setTimeout(r, 10 + (i % 3))) + + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + return sink.digestHex() + }) + ) + + // Start second batch while first is still running + await new Promise((r) => setTimeout(r, 20)) + + const batch2Promise = Promise.all( + Array.from({ length: batchSize }, async (_, i) => { + const payload = randBuf(limit * 3 + i) + const sink = new HashSpillWritable({ limitInMemoryBytes: limit, tmpRoot }) + await pipeline(readableFrom(payload), sink) + + await pipeline( + sink.toReadable({ autoCleanup: true }), + new Writable({ + write(_c, _e, cb) { + cb() + }, + }) + ) + + return sink.digestHex() + }) + ) + + const [results1, results2] = await Promise.all([batch1Promise, batch2Promise]) + + expect(results1).toHaveLength(batchSize) + expect(results2).toHaveLength(batchSize) + + // Allow all cleanup to complete + await new Promise((r) => setTimeout(r, 50)) + expect(await countHashspillDirs(tmpRoot)).toBe(0) + }) +}) diff --git a/src/test/vectors.test.ts b/src/test/vectors.test.ts new file mode 100644 index 00000000..ca2d9765 --- /dev/null +++ b/src/test/vectors.test.ts @@ -0,0 +1,309 @@ +'use strict' + +import app from '../app' +import { getConfig } from '../config' +import { FastifyInstance } from 'fastify' +import { useMockObject, useMockQueue } from './common' +import { + CreateIndexCommandOutput, + DeleteVectorsOutput, + GetVectorsCommandOutput, + ListVectorsOutput, + PutVectorsOutput, + QueryVectorsOutput, +} from '@aws-sdk/client-s3vectors' +import { KnexVectorMetadataDB, VectorStoreManager } from '@storage/protocols/vector' +import { useStorage } from './utils/storage' + +const { serviceKeyAsync } = getConfig() + +let appInstance: FastifyInstance +let serviceToken: string + +// Use the common mock helpers +useMockObject() +useMockQueue() + +jest.mock('@storage/protocols/vector/adapter/s3-vector') + +const storageTest = useStorage() +const mockVectorStore = { + deleteVectorIndex: jest.fn().mockResolvedValue({} as CreateIndexCommandOutput), + createVectorIndex: jest.fn().mockResolvedValue({} as CreateIndexCommandOutput), + putVectors: jest.fn().mockResolvedValue({} as PutVectorsOutput), + listVectors: jest.fn().mockResolvedValue({} as ListVectorsOutput), + queryVectors: jest.fn().mockResolvedValue({} as QueryVectorsOutput), + deleteVectors: jest.fn().mockResolvedValue({} as DeleteVectorsOutput), + getVectors: jest.fn().mockResolvedValue({} as GetVectorsCommandOutput), +} + +beforeEach(async () => { + jest.clearAllMocks() + + appInstance = app() + + // Create service role token + serviceToken = await serviceKeyAsync + + // Create real S3Vector instance with mocked client and mock DB + const mockVectorDB = new KnexVectorMetadataDB(storageTest.database.connection.pool.acquire()) + const s3Vector = new VectorStoreManager(mockVectorStore, mockVectorDB, { + tenantId: 'test-tenant', + vectorBucketName: 'test-bucket', + }) + + // Decorate fastify instance with real S3Vector + appInstance.decorate('s3Vector', s3Vector) +}) + +afterEach(async () => { + await appInstance.close() + await storageTest.database.connection.dispose() +}) + +describe('POST /vectors/CreateIndex', () => { + const validCreateIndexRequest = { + dataType: 'float32', + dimension: 1536, + distanceMetric: 'cosine', + indexName: 'test-index', + vectorBucketName: 'test-bucket', + metadataConfiguration: { + nonFilterableMetadataKeys: ['key1', 'key2'], + }, + } + + it('should create vector index successfully with valid request', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(200) + + // Verify the CreateIndexCommand was called with correct parameters including tenantId prefix + const createIndexCommand = mockVectorStore.createVectorIndex.mock.calls[0][0] as unknown as { + input: Record + } + expect(createIndexCommand.input.indexName).toBe('test-tenant-test-index') + expect(createIndexCommand.input.dataType).toBe('float32') + expect(createIndexCommand.input.dimension).toBe(1536) + }) + + it('should require authentication with service role', async () => { + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(401) + // Vector service not called when validation fails + }) + + it('should reject request with invalid JWT role', async () => { + const invalidToken = 'invalid-token' + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${invalidToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(403) + // Vector service not called when validation fails + }) + + it('should validate required fields', async () => { + const incompleteRequest = { + dataType: 'float32', + dimension: 1536, + // missing required fields + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: incompleteRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate dataType enum', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + dataType: 'invalid-type', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate distanceMetric enum', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + distanceMetric: 'invalid-metric', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate dimension is a number', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + dimension: 'not-a-number', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should validate metadataConfiguration structure', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + metadataConfiguration: { + // missing required nonFilterableMetadataKeys + invalidKey: 'value', + }, + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) + + it('should handle vector service not configured', async () => { + // Mock app without s3Vector service + const appWithoutVector = app() + + const response = await appWithoutVector.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(400) + const body = JSON.parse(response.body) + expect(body.error).toBe('FeatureNotEnabled') + expect(body.message).toBe('Vector service not configured') + + await appWithoutVector.close() + }) + + it('should handle S3Vector service errors', async () => { + const s3Error = new Error('S3VectorsClient error') + // Mock error - need to cast to bypass type restrictions + mockVectorStore.createVectorIndex.mockRejectedValue(s3Error) + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: validCreateIndexRequest, + }) + + expect(response.statusCode).toBe(500) + expect(mockVectorStore.createVectorIndex).toHaveBeenCalledTimes(1) + }) + + it('should accept valid request without optional metadataConfiguration', async () => { + const requestWithoutMetadata = { + dataType: 'float32' as const, + dimension: 1536, + distanceMetric: 'euclidean' as const, + indexName: 'test-index-2', + vectorBucketName: 'test-bucket-2', + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: requestWithoutMetadata, + }) + + expect(response.statusCode).toBe(200) + expect(mockVectorStore.createVectorIndex).toHaveBeenCalledTimes(1) + + // Verify the CreateIndexCommand was called with correct parameters + const createIndexCommand = mockVectorStore.createVectorIndex.mock.calls[0][1] + expect(createIndexCommand.input.indexName).toBe('test-tenant-test-index-2') + expect(createIndexCommand.input.distanceMetric).toBe('euclidean') + }) + + it('should validate nonFilterableMetadataKeys as array of strings', async () => { + const invalidRequest = { + ...validCreateIndexRequest, + metadataConfiguration: { + nonFilterableMetadataKeys: ['valid', 123, 'another-valid'], + }, + } + + const response = await appInstance.inject({ + method: 'POST', + url: '/vectors/CreateIndex', + headers: { + authorization: `Bearer ${serviceToken}`, + }, + payload: invalidRequest, + }) + + expect(response.statusCode).toBe(400) + // Vector service not called when validation fails + }) +})