Improvement/cldsrv 724 backbeat related functional tests

.github/workflows/tests.yaml

@@ -251,6 +251,8 @@ jobs:
       S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
       S3DATA: multiple
       S3METADATA: mongodb
+      AWS_REGION: us-east-1
+      AWS_DEFAULT_REGION: us-east-1
       ENABLE_NULL_VERSION_COMPAT_MODE: true # needed with mongodb backend
       JOB_NAME: ${{ github.job }}
     steps:

examples/node-md-search.js

@@ -1,28 +1,46 @@
-const { S3 } = require('aws-sdk');
+const { S3Client, ListObjectsCommand } = require('@aws-sdk/client-s3');
+const { NodeHttpHandler } = require('@smithy/node-http-handler');
+const http = require('http');
+
 const config = {
-    sslEnabled: false,
     endpoint: 'http://127.0.0.1:8000',
-    signatureCache: false,
-    signatureVersion: 'v4',
     region: 'us-east-1',
-    s3ForcePathStyle: true,
-    accessKeyId: 'accessKey1',
-    secretAccessKey: 'verySecretKey1',
+    forcePathStyle: true,
+    credentials: {
+        accessKeyId: 'accessKey1',
+        secretAccessKey: 'verySecretKey1',
+    },
+    requestHandler: new NodeHttpHandler({
+        httpAgent: new http.Agent({ keepAlive: false }),
+    }),
 };
-const s3Client = new S3(config);
 
-const encodedSearch =
-    encodeURIComponent('x-amz-meta-color="blue"');
-const req = s3Client.listObjects({ Bucket: 'bucketname' });
+const s3Client = new S3Client(config);
+
+const encodedSearch = encodeURIComponent('x-amz-meta-color="blue"');
+
+const command = new ListObjectsCommand({ Bucket: 'bucketname' });
+
+command.middlewareStack.add(
+    next => async args => {
+        if (args.request && args.request.path) {
+            // eslint-disable-next-line no-param-reassign
+            args.request.path = `${args.request.path}?search=${encodedSearch}`;
+        }
+        return next(args);
+    },
+    {
+        step: 'build',
+        name: 'addSearchParameter',
+        priority: 'high'
+    }
+);
 
-// the build event
-req.on('build', () => {
-    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
-});
-req.on('success', res => {
-    process.stdout.write(`Result ${res.data}`);
-});
-req.on('error', err => {
-    process.stdout.write(`Error ${err}`);
-});
-req.send();
+// Send command and handle response
+s3Client.send(command)
+    .then(data => {
+        process.stdout.write(`Result ${JSON.stringify(data)}`);
+    })
+    .catch(err => {
+        process.stdout.write(`Error ${err}`);
+    });

lib/api/listParts.js

@@ -89,6 +89,7 @@ function listParts(authInfo, request, log, callback) {
     if (maxParts > constants.listingHardLimit) {
         maxParts = constants.listingHardLimit;
     }
+
     const partNumberMarker =
         Number.parseInt(request.query['part-number-marker'], 10) ?
         Number.parseInt(request.query['part-number-marker'], 10) : 0;

lib/data/wrapper.js

@@ -27,6 +27,13 @@ if (config.backends.data === 'mem') {
     client = new DataFileInterface(config);
     implName = 'file';
 } else if (config.backends.data === 'multiple') {
+        Object.keys(config.locationConstraints).filter(k => 
+            config.locationConstraints[k].type === 'aws_s3'
+        ).map(k => ({
+            name: k,
+            region: config.locationConstraints[k].details.region,
+            https: config.locationConstraints[k].details.https
+        }));
     const clients = parseLC(config, vault);
     client = new MultipleBackendGateway(
         clients, metadata, locationStorageCheck);

package.json

@@ -19,7 +19,9 @@
   },
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
+    "@aws-sdk/client-iam": "^3.930.0",
     "@aws-sdk/client-s3": "^3.908.0",
+    "@aws-sdk/client-sts": "^3.930.0",
     "@aws-sdk/credential-providers": "^3.864.0",
     "@aws-sdk/middleware-retry": "^3.374.0",
     "@aws-sdk/protocol-http": "^3.374.0",
@@ -28,7 +30,7 @@
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
     "@smithy/node-http-handler": "^3.0.0",
-    "arsenal": "git+https://github.com/scality/Arsenal#8.2.35",
+    "arsenal": "git+https://github.com/scality/Arsenal#cf48ed18c670ce69f14a67beddf31443bf59a628",
     "async": "2.6.4",
     "bucketclient": "scality/bucketclient#8.2.7",
     "bufferutil": "^4.0.8",
@@ -69,7 +71,7 @@
     "istanbul": "^0.4.5",
     "istanbul-api": "^3.0.0",
     "lolex": "^6.0.0",
-    "mocha": "^10.8.2",
+    "mocha": "^11.7.5",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",
     "node-mocks-http": "^1.16.1",
@@ -89,7 +91,7 @@
   "scripts": {
     "cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
     "dev": "nodemon --exec \"yarn run start\"",
-    "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit",
+    "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test --exit",
     "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit",
     "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket --exit",
     "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support --exit",
@@ -135,7 +137,7 @@
     "test_legacy_location": "CI=true S3QUOTA=scuba S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit",
     "test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi --exit",
     "test_sur": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/sur --exit",
-    "multiple_backend_test": "CI=true S3BACKEND=mem S3METADATA=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend --exit",
+    "multiple_backend_test": "CI=true S3BACKEND=mem S3METADATA=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend/ --exit",
     "cover": "nyc --clean --silent yarn run",
     "postcover": "nyc report --report-dir ./coverage/test --reporter=lcov"
   }