migrate verifyReplication to aws sdk v3

Issue : S3UTILS-201

Author: SylvainSenechal

VerifyReplication/storage/s3.js

@@ -1,4 +1,6 @@
-const AWS = require('aws-sdk');
+const { S3Client, HeadObjectCommand, ListObjectsV2Command } = require('@aws-sdk/client-s3');
+const { NodeHttpHandler } = require('@aws-sdk/node-http-handler');
+const { ConfiguredRetryStrategy } = require('@smithy/util-retry');
 const fs = require('fs');
 const http = require('http');
 const https = require('https');
@@ -19,7 +21,6 @@ function getClient(params) {
     } = params;
     const s3EndpointIsHttps = (endpoint && endpoint.startsWith('https:')) || false;
     let agent;
-    let clientLogger;
 
     if (s3EndpointIsHttps) {
         agent = new https.Agent({
@@ -31,61 +32,54 @@ function getClient(params) {
         agent = new http.Agent({ keepAlive: true });
     }
 
-    // enable/disable sdk logs
-    if (showClientLogsIfAvailable) {
-        // TODO: may be use werelogs
-        clientLogger = console;
-    }
+    // Configure retry strategy to match original AWS SDK v2 configuration
+    // Options specific to s3 requests - maxRetries & customBackoff
+    // Default aws sdk retry count is 3 with an exponential delay of 2^n * 30 ms
+    const retryStrategy = new ConfiguredRetryStrategy(
+        defaults.AWS_SDK_REQUEST_RETRIES,
+        (retryCount) => {
+            // retry with exponential backoff delay capped at 60s max
+            // between retries, and a little added jitter
+            const backoff = Math.min(defaults.AWS_SDK_REQUEST_INITIAL_DELAY_MS
+                * 2 ** retryCount, defaults.AWS_SDK_REQUEST_MAX_BACKOFF_LIMIT_MS)
+                * (0.9 + Math.random() * 0.2);
+            // show retry errors only if client logs are enabled as this may
+            // increase log size!
+            if (showClientLogsIfAvailable) {
+                log.error('awssdk request error', {
+                    retryCount,
+                    backoff,
+                });
+            }
+            return backoff;
+        }
+    );
 
-    const options = {
-        accessKeyId: accessKey,
-        secretAccessKey: secretKey,
-        endpoint,
-        region,
-        sslEnabled: s3EndpointIsHttps,
-        s3ForcePathStyle: true,
-        apiVersions: { s3: '2006-03-01' },
-        signatureVersion: 'v4',
-        signatureCache: false,
-        httpOptions: {
-            timeout: httpTimeout,
-            agent,
+    const clientConfig = {
+        region: region,
+        credentials: {
+            accessKeyId: accessKey,
+            secretAccessKey: secretKey,
         },
-        logger: clientLogger,
+        endpoint,
+        forcePathStyle: true,
+        retryStrategy,
+        requestHandler: new NodeHttpHandler({
+            httpAgent: s3EndpointIsHttps ? undefined : agent,
+            httpsAgent: s3EndpointIsHttps ? agent : undefined,
+            requestTimeout: httpTimeout || 300000,
+        }),
     };
 
-    /**
-     *  Options specific to s3 requests
-     *  `maxRetries` & `customBackoff` are set only to s3 requests
-     *  default aws sdk retry count is 3 with an exponential delay of 2^n * 30 ms
-     */
-    const s3Options = {
-        maxRetries: defaults.AWS_SDK_REQUEST_RETRIES,
-        retryDelayOptions: {
-            customBackoff: (retryCount, error) => {
-                // retry with exponential backoff delay capped at 60s max
-                // between retries, and a little added jitter
-                const backoff = Math.min(defaults.AWS_SDK_REQUEST_INITIAL_DELAY_MS
-                    * 2 ** retryCount, defaults.AWS_SDK_REQUEST_MAX_BACKOFF_LIMIT_MS)
-                    * (0.9 + Math.random() * 0.2);
-                // show retry errors only if client logs are enabled as this may
-                // increase log size!
-                if (showClientLogsIfAvailable) {
-                    log.error('awssdk request error', {
-                        error,
-                        retryCount,
-                        backoff,
-                    });
-                }
-                return backoff;
-            },
-        },
-    };
+    // Add logger if available
+    if (showClientLogsIfAvailable) {
+        clientConfig.logger = console;
+    }
 
-    return new AWS.S3({ ...options, ...s3Options });
+    return new S3Client(clientConfig);
 }
 
-function getObjMd(params, cb) {
+async function getObjMd(params, cb) {
     const {
         client,
         bucket,
@@ -98,31 +92,29 @@ function getObjMd(params, cb) {
         return cb(new Error(errMsg));
     }
 
-    return client.headObject({
-        Bucket: bucket,
-        Key: key,
-        VersionId: versionId,
-    }, (err, data) => {
-        if (err) {
-            return cb(err);
+    try {
+        const commandParams = {
+            Bucket: bucket,
+            Key: key,
+        };
+        if (versionId) {
+            commandParams.VersionId = versionId;
         }
-        const {
-            ContentLength,
-            LastModified,
-            VersionId,
-            Metadata,
-        } = data;
+        
+        const data = await client.send(new HeadObjectCommand(commandParams));
         const resp = {
-            size: ContentLength,
-            lastModified: LastModified,
-            versionId: VersionId,
-            md: Metadata,
+            size: data.ContentLength,
+            lastModified: data.LastModified,
+            versionId: data.VersionId,
+            md: data.Metadata,
         };
         return cb(null, resp);
-    });
+    } catch (err) {
+        return cb(err);
+    }
 }
 
-function listObjects(params, cb) {
+async function listObjects(params, cb) {
     const {
         client,
         bucket,
@@ -140,14 +132,19 @@ function listObjects(params, cb) {
         return cb(new Error(errMsg));
     }
 
-    // TODO: support listing all versions
-    return client.listObjectsV2({
-        Bucket: bucket,
-        MaxKeys: listingLimit,
-        Prefix: prefix,
-        Delimiter: delimiter,
-        ContinuationToken: nextContinuationToken,
-    }, cb);
+    try {
+        // TODO: support listing all versions
+        const data = await client.send(new ListObjectsV2Command({
+            Bucket: bucket,
+            MaxKeys: listingLimit,
+            Prefix: prefix,
+            Delimiter: delimiter,
+            ContinuationToken: nextContinuationToken,
+        }));
+        return cb(null, data);
+    } catch (err) {
+        return cb(err);
+    }
 }
 
 module.exports = {

VerifyReplication/verifyReplication.js

@@ -1,6 +1,7 @@
 const async = require('async');
 
 const storage = require('./storage');
+const { NotFound } = require('@aws-sdk/client-s3');
 
 let bucketMatch = false;
 let compareSize = false;
@@ -41,26 +42,20 @@ function verifyObjects(objectList, cb) {
 
         return destinationStorage.getObjMd(params, (err, dstMd) => {
             ++statusObj.dstProcessedCount;
-            if (err && err.code !== 'NotFound') {
+            if (err) {
                 ++statusObj.dstFailedMdRetrievalsCount;
                 logger.error('error getting metadata', {
                     error: err,
                     bucket: statusObj.dstBucket,
                     key: dstKey,
                     srcLastModified,
                 });
+                if (err instanceof NotFound) {
+                    ++statusObj.missingInDstCount;
+                }
                 // log the error and continue processing objects
                 return done();
             }
-            if (err && err.code === 'NotFound') {
-                ++statusObj.missingInDstCount;
-                logger.info('object missing in destination', {
-                    key,
-                    size,
-                    srcLastModified,
-                });
-                return done();
-            }
             const srcSize = Number.parseInt(size, 10);
             const dstSize = Number.parseInt(dstMd.size, 10);
             if (compareSize && (srcSize !== dstSize)) {
@@ -81,6 +76,9 @@ function verifyObjects(objectList, cb) {
 }
 
 function handlePrefixes(prefixList, cb) {
+    if (!prefixList || prefixList.length === 0) {
+        return process.nextTick(cb);
+    }
     const prefixes = prefixList.map(p => p.Prefix);
     return async.eachLimit(prefixes, listingWorkers, (prefix, done) => {
         const params = {
@@ -106,7 +104,7 @@ function listAndCompare(params, cb) {
         }
         const {
             IsTruncated,
-            NextContinuationToken: nextContinuationToken,
+            NextContinuationToken,
             Contents,
             CommonPrefixes,
         } = data;
@@ -118,7 +116,7 @@ function listAndCompare(params, cb) {
                 return cb(error);
             }
             if (IsTruncated) {
-                const listingParams = { ...params, nextContinuationToken };
+                const listingParams = { ...params, nextContinuationToken: NextContinuationToken };
                 return listAndCompare(listingParams, cb);
             }
             logger.info('completed listing and compare', {