Merge branch 'improvement/S3UTILS-199' into q/1.16

Author: bert-e

CRR/ReplicationStatusUpdater.js

@@ -1,9 +1,12 @@
 const {
-    doWhilst, eachSeries, eachLimit, waterfall, series,
+    doWhilst, eachSeries, eachLimit, waterfall,
 } = require('async');
 const { ObjectMD } = require('arsenal').models;
-
 const { setupClients } = require('./clients');
+const { 
+    ListObjectVersionsCommand, 
+    GetBucketReplicationCommand 
+} = require('@aws-sdk/client-s3');
 
 const LOG_PROGRESS_INTERVAL_MS = 10000;
 
@@ -26,6 +29,7 @@ class ReplicationStatusUpdater {
      * @param {number} [params.maxScanned] - (Optional) Maximum number of items to scan.
      * @param {string} [params.keyMarker] - (Optional) Key marker for resuming object listing.
      * @param {string} [params.versionIdMarker] - (Optional) Version ID marker for resuming object listing.
+     * @param {boolean} [params.currentVersionOnly] - (Optional) Whether to process only the current version of objects.
      */
     constructor(params, log) {
         const {
@@ -266,13 +270,15 @@ class ReplicationStatusUpdater {
      * @returns {void}
      */
     _listObjectVersions(bucket, VersionIdMarker, KeyMarker, cb) {
-        return this.s3.listObjectVersions({
+        this.s3.send(new ListObjectVersionsCommand({
             Bucket: bucket,
             MaxKeys: this.listingLimit,
             Prefix: this.targetPrefix,
             VersionIdMarker,
             KeyMarker,
-        }, cb);
+        }))
+            .then(data => cb(null, data))
+            .catch(cb);
     }
 
     /**
@@ -284,15 +290,16 @@ class ReplicationStatusUpdater {
      * @returns {void}
      */
     _markPending(bucket, versions, cb) {
-        const options = { Bucket: bucket };
         waterfall([
-            next => this.s3.getBucketReplication(options, (err, res) => {
-                if (err) {
+            async () => {
+                try {
+                    const res = await this.s3.send(new GetBucketReplicationCommand({ Bucket: bucket }));
+                    return res.ReplicationConfiguration;
+                } catch (err) {
                     this.log.error('error getting bucket replication', { error: err });
-                    return next(err);
+                    throw err;
                 }
-                return next(null, res.ReplicationConfiguration);
-            }),
+            },
             (repConfig, next) => {
                 const { Rules } = repConfig;
                 const storageClass = this.siteName || Rules[0].Destination.StorageClass;
@@ -348,7 +355,8 @@ class ReplicationStatusUpdater {
                         this.log.error('error listing object versions', { error: err });
                         return done(err);
                     }
-                    return this._markPending(bucket, data.Versions.concat(data.DeleteMarkers), err => {
+                    const versions = (data.Versions || []).concat(data.DeleteMarkers || []);
+                    return this._markPending(bucket, versions, err => {
                         if (err) {
                             return done(err);
                         }

CRR/clients.js

@@ -1,10 +1,12 @@
-const AWS = require('aws-sdk');
+const { S3Client } = require('@aws-sdk/client-s3');
+const { NodeHttpHandler } = require('@aws-sdk/node-http-handler');
+const { ConfiguredRetryStrategy } = require('@smithy/util-retry');
 const http = require('http');
+const https = require('https');
 const BackbeatClient = require('../BackbeatClient');
 
 const AWS_SDK_REQUEST_DELAY_MS = 30;
 const AWS_SDK_REQUEST_RETRIES = 100;
-const LOG_PROGRESS_INTERVAL_MS = 10000;
 
 /**
  * Sets up and configures AWS S3 and Backbeat clients.
@@ -27,7 +29,37 @@ function setupClients({
     secretKey,
     endpoint,
 }, log) {
-    const awsConfig = {
+    const awsv3Config = {
+        region: 'us-east-1',
+        credentials: {
+            accessKeyId: accessKey,
+            secretAccessKey: secretKey,
+        },
+        endpoint,
+        forcePathStyle: true,
+        requestHandler: new NodeHttpHandler({
+            httpAgent: new http.Agent({ keepAlive: true }),
+            httpsAgent: new https.Agent({ 
+                keepAlive: true,
+                // With SDK v2, SSL was disabled. Here we added this
+                // to be able to run the script locally on a lab for testing,
+                // so https will work but there will be no security enforced
+                rejectUnauthorized: false
+            }),
+            requestTimeout: 60000,
+        }),
+        retryStrategy: new ConfiguredRetryStrategy(
+            AWS_SDK_REQUEST_RETRIES, // maxAttempts
+            attempt => {
+                log.error('aws sdk request error', { retryCount: attempt });
+                // The delay is not truly exponential; it resets to the minimum after every 10 calls,
+                // with a maximum delay of 15 seconds.
+                return AWS_SDK_REQUEST_DELAY_MS * (2 ** (attempt % 10));
+            }
+        )
+    };
+
+    const awsv2Config = {
         accessKeyId: accessKey,
         secretAccessKey: secretKey,
         endpoint,
@@ -38,37 +70,15 @@ function setupClients({
         signatureVersion: 'v4',
         signatureCache: false,
         httpOptions: {
-            timeout: 0,
+            timeout: 60000,
             agent: new http.Agent({ keepAlive: true }),
         },
     };
 
-    /**
-     * Custom backoff strategy for AWS SDK requests.
-     * @param {number} retryCount - The current retry attempt.
-     * @param {Error} error - The error that caused the retry.
-     * @returns {number} The delay in milliseconds before the next retry.
-     */
-    function customBackoffStrategy(retryCount, error) {
-        this.log.error('aws sdk request error', { error, retryCount });
-        // The delay is not truly exponential; it resets to the minimum after every 10 calls,
-        // with a maximum delay of 15 seconds.
-        return AWS_SDK_REQUEST_DELAY_MS * (2 ** (retryCount % 10));
-    }
-
-    // Specific options for S3 requests
-    const s3SpecificOptions = {
-        maxRetries: AWS_SDK_REQUEST_RETRIES,
-        customBackoff: customBackoffStrategy,
+    return {
+        s3: new S3Client(awsv3Config),
+        bb: new BackbeatClient(awsv2Config),
     };
-
-    // Create an S3 client instance
-    const s3 = new AWS.S3({ ...awsConfig, ...s3SpecificOptions });
-
-    // Create a BackbeatClient instance
-    const bb = new BackbeatClient(awsConfig);
-
-    return { s3, bb };
 }
 
 module.exports = { setupClients };

package.json

@@ -26,7 +26,10 @@
   },
   "homepage": "https://github.com/scality/s3utils#readme",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.873.0",
+    "@aws-sdk/node-http-handler": "^3.374.0",
     "@senx/warp10": "^2.0.3",
+    "@smithy/util-retry": "^4.0.7",
     "JSONStream": "^1.3.5",
     "arsenal": "git+https://github.com/scality/arsenal#8.2.26",
     "async": "^3.2.6",