Migrate aws sdk from v2 to v3 for CRR

Issue: S3UTILS-199

Author: SylvainSenechal

CRR/ReplicationStatusUpdater.js

@@ -1,9 +1,12 @@
 const {
-    doWhilst, eachSeries, eachLimit, waterfall, series,
+    doWhilst, eachSeries, eachLimit, waterfall,
 } = require('async');
 const { ObjectMD } = require('arsenal').models;
-
 const { setupClients } = require('./clients');
+const { 
+    ListObjectVersionsCommand, 
+    GetBucketReplicationCommand 
+} = require('@aws-sdk/client-s3');
 
 const LOG_PROGRESS_INTERVAL_MS = 10000;
 
@@ -26,6 +29,7 @@ class ReplicationStatusUpdater {
      * @param {number} [params.maxScanned] - (Optional) Maximum number of items to scan.
      * @param {string} [params.keyMarker] - (Optional) Key marker for resuming object listing.
      * @param {string} [params.versionIdMarker] - (Optional) Version ID marker for resuming object listing.
+     * @param {boolean} [params.currentVersionOnly] - (Optional) Whether to process only the current version of objects.
      */
     constructor(params, log) {
         const {
@@ -266,13 +270,17 @@ class ReplicationStatusUpdater {
      * @returns {void}
      */
     _listObjectVersions(bucket, VersionIdMarker, KeyMarker, cb) {
-        return this.s3.listObjectVersions({
+        this.s3.send(new ListObjectVersionsCommand({
             Bucket: bucket,
             MaxKeys: this.listingLimit,
             Prefix: this.targetPrefix,
             VersionIdMarker,
             KeyMarker,
-        }, cb);
+        }))
+            .then(data => {
+                cb(null, data);
+            })
+            .catch(err => cb(err));
     }
 
     /**
@@ -284,15 +292,17 @@ class ReplicationStatusUpdater {
      * @returns {void}
      */
     _markPending(bucket, versions, cb) {
-        const options = { Bucket: bucket };
         waterfall([
-            next => this.s3.getBucketReplication(options, (err, res) => {
-                if (err) {
-                    this.log.error('error getting bucket replication', { error: err });
-                    return next(err);
-                }
-                return next(null, res.ReplicationConfiguration);
-            }),
+            next => {
+                this.s3.send(new GetBucketReplicationCommand(
+                    { Bucket: bucket },
+                ))
+                    .then(res => next(null, res.ReplicationConfiguration))
+                    .catch(err => {
+                        this.log.error('error getting bucket replication', { error: err });
+                        return next(err);
+                    });
+            },
             (repConfig, next) => {
                 const { Rules } = repConfig;
                 const storageClass = this.siteName || Rules[0].Destination.StorageClass;
@@ -348,7 +358,8 @@ class ReplicationStatusUpdater {
                         this.log.error('error listing object versions', { error: err });
                         return done(err);
                     }
-                    return this._markPending(bucket, data.Versions.concat(data.DeleteMarkers), err => {
+                    const versions = (data.Versions || []).concat(data.DeleteMarkers || []);
+                    return this._markPending(bucket, versions, err => {
                         if (err) {
                             return done(err);
                         }

CRR/clients.js

@@ -1,10 +1,12 @@
-const AWS = require('aws-sdk');
+const { S3Client } = require('@aws-sdk/client-s3');
+const { NodeHttpHandler } = require('@aws-sdk/node-http-handler');
+const { ConfiguredRetryStrategy } = require('@smithy/util-retry');
 const http = require('http');
+const https = require('https');
 const BackbeatClient = require('../BackbeatClient');
 
 const AWS_SDK_REQUEST_DELAY_MS = 30;
 const AWS_SDK_REQUEST_RETRIES = 100;
-const LOG_PROGRESS_INTERVAL_MS = 10000;
 
 /**
  * Sets up and configures AWS S3 and Backbeat clients.
@@ -27,7 +29,34 @@ function setupClients({
     secretKey,
     endpoint,
 }, log) {
-    const awsConfig = {
+    const awsv3Config = {
+        region: 'us-east-1',
+        credentials: {
+            accessKeyId: accessKey,
+            secretAccessKey: secretKey,
+        },
+        endpoint,
+        forcePathStyle: true,
+        requestHandler: new NodeHttpHandler({
+            httpAgent: new http.Agent({ keepAlive: true }),
+            httpsAgent: new https.Agent({ 
+                keepAlive: true,
+                rejectUnauthorized: false
+            }),
+            requestTimeout: 0,
+        }),
+        retryStrategy: new ConfiguredRetryStrategy(
+            AWS_SDK_REQUEST_RETRIES, // maxAttempts
+            attempt => {
+                log.error('aws sdk request error', { retryCount: attempt });
+                // The delay is not truly exponential; it resets to the minimum after every 10 calls,
+                // with a maximum delay of 15 seconds.
+                return AWS_SDK_REQUEST_DELAY_MS * (2 ** (attempt % 10));
+            }
+        )
+    };
+
+    const awsv2Config = {
         accessKeyId: accessKey,
         secretAccessKey: secretKey,
         endpoint,
@@ -43,32 +72,10 @@ function setupClients({
         },
     };
 
-    /**
-     * Custom backoff strategy for AWS SDK requests.
-     * @param {number} retryCount - The current retry attempt.
-     * @param {Error} error - The error that caused the retry.
-     * @returns {number} The delay in milliseconds before the next retry.
-     */
-    function customBackoffStrategy(retryCount, error) {
-        this.log.error('aws sdk request error', { error, retryCount });
-        // The delay is not truly exponential; it resets to the minimum after every 10 calls,
-        // with a maximum delay of 15 seconds.
-        return AWS_SDK_REQUEST_DELAY_MS * (2 ** (retryCount % 10));
-    }
-
-    // Specific options for S3 requests
-    const s3SpecificOptions = {
-        maxRetries: AWS_SDK_REQUEST_RETRIES,
-        customBackoff: customBackoffStrategy,
+    return {
+        s3: new S3Client(awsv3Config),
+        bb: new BackbeatClient(awsv2Config),
     };
-
-    // Create an S3 client instance
-    const s3 = new AWS.S3({ ...awsConfig, ...s3SpecificOptions });
-
-    // Create a BackbeatClient instance
-    const bb = new BackbeatClient(awsConfig);
-
-    return { s3, bb };
 }
 
 module.exports = { setupClients };

package.json

@@ -26,7 +26,10 @@
   },
   "homepage": "https://github.com/scality/s3utils#readme",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.873.0",
+    "@aws-sdk/node-http-handler": "^3.374.0",
     "@senx/warp10": "^2.0.3",
+    "@smithy/util-retry": "^4.0.7",
     "JSONStream": "^1.3.5",
     "arsenal": "git+https://github.com/scality/arsenal#8.2.26",
     "async": "^3.2.6",