Improvement/s3 utils 171 bump deps

.github/workflows/test.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Install node
         uses: actions/setup-node@v4
         with:
-          node-version: '16'
+          node-version: '22'
           cache: 'yarn'
       - run: sudo apt-get update -q
       # Ubuntu 24.04 doesn't have libssl1.1 which is needed by mongo-memory-server
@@ -73,7 +73,7 @@ jobs:
       - run: yarn --silent test:unit
       - run: yarn --silent test:functional
         env:
-          MONGODB_REPLICASET: "localhost:27018"
+          MONGODB_REPLICASET: "127.0.0.1:27018"
       - name: "Debug: SSH to runner"
         uses: scality/actions/action-ssh-to-runner@1.9.0
         continue-on-error: true

CRR/ReplicationStatusUpdater.js

@@ -352,7 +352,7 @@ class ReplicationStatusUpdater {
                     });
                 },
             ),
-            () => {
+            async () => {
                 if (this._nUpdated >= this.maxUpdates || this._nProcessed >= this.maxScanned) {
                     this._logProgress();
                     let remainingBuckets;

CompareRaftMembers/BlockDigestsStorage.js

@@ -27,7 +27,7 @@ class BlockDigestsStorage extends stream.Writable {
     constructor(params) {
         super({ objectMode: true });
         const { levelPath, db } = params;
-        if (params.levelPath) {
+        if (levelPath) {
             this.db = new Level(params.levelPath);
         } else {
             this.db = db;
@@ -38,16 +38,16 @@ class BlockDigestsStorage extends stream.Writable {
     }
 
     _write(blockInfo, encoding, callback) {
-        const { size, digest } = blockInfo;
+        const { size, digest, lastKey } = blockInfo;
         this.cargo.push({
             type: 'put',
-            key: blockInfo.lastKey, // index by last block key for efficient lookup
+            key: lastKey, // index by last block key for efficient lookup
             value: JSON.stringify({ size, digest }),
         });
         // heuristic to have basic flow control: delay the callback
         // while queue size is above a reasonable size
         async.whilst(
-            () => this.cargo.length() > MAX_QUEUE_SIZE,
+            async () => this.cargo.length() > MAX_QUEUE_SIZE,
             cb => setTimeout(cb, 100),
             () => callback(),
         );
@@ -57,9 +57,9 @@ class BlockDigestsStorage extends stream.Writable {
         if (this.cargo.idle()) {
             this.db.close(callback);
         } else {
-            this.cargo.drain = () => {
+            this.cargo.drain(() => {
                 this.db.close(callback);
-            };
+            });
         }
     }
 }

CountItems/CountManager.js

@@ -174,7 +174,9 @@ class CountManager {
         }));
         this.store.buckets += bucketCount;
         this.store.bucketList = this.store.bucketList.concat(transformedInfos);
-        this.q.push(bucketInfos);
+        bucketInfos.forEach(bucketInfo => {
+            this.q.push(bucketInfo);
+        });
         this.log.debug('added work', {
             workInQueue: this.q.length(),
             workInProgress: this.q.running(),
@@ -195,18 +197,22 @@ class CountManager {
             Object.values(this.workers)
                 .forEach(worker => this.workerList.push(worker.id));
         }
-        this.q.error = err => {
+        this.q.error(err => {
+            this.log.error('error processing bucket', {
+                error: err,
+                method: 'CountManager::addWork',
+            });
             this.q.pause();
             this.q.kill();
             return process.nextTick(onceCB, err);
-        };
-        this.q.drain = () => {
+        });
+        this.q.drain(() => {
             if (this.q.idle()) {
                 this.q.pause();
                 this.q.kill();
                 process.nextTick(onceCB);
             }
-        };
+        });
         this.q.resume();
     }
 }

CountItems/CountWorker.js

@@ -36,7 +36,7 @@ class CountWorker {
         if (bucketInfoObj._websiteConfiguration) {
             Object.assign(bucketInfoObj, { _websiteConfiguration: null });
         }
-        const bucketInfo = BucketInfo.fromObj(bucketInfoObj);
+        const bucketInfo = BucketInfo.fromObj(bucketInfoObj.bucketInfo || bucketInfoObj);
         const bucketName = bucketInfo.getName();
         this.log.info(`${process.pid} handling ${bucketName}`);
         return async.waterfall([

CountItems/CountWorkerObj.js

@@ -1,4 +1,4 @@
-const uuid = require('node-uuid');
+const { v4: uuid } = require('uuid');
 const { once } = require('arsenal').jsutil;
 const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
 
@@ -21,9 +21,13 @@ class CountWorkerObj {
             }
         });
         this._worker.on('message', data => {
-            if (data.owner !== 'scality') return;
+            if (data.owner !== 'scality') {
+                return;
+            }
             const cb = this._getCallback(data.id);
-            if (!cb) return;
+            if (!cb) {
+                return;
+            }
             switch (data.type) {
             case 'setup':
                 if (data.status === 'passed') {
@@ -81,7 +85,9 @@ class CountWorkerObj {
     }
 
     _getCallback(id) {
-        if (!this.callbacks.has(id)) return null;
+        if (!this.callbacks.has(id)) {
+            return null;
+        }
         const ret = this.callbacks.get(id);
         this.callbacks.delete(id);
         return ret.callback;
@@ -95,7 +101,7 @@ class CountWorkerObj {
     }
 
     setup(callback) {
-        const id = uuid.v4();
+        const id = uuid();
         this._addCallback(id, 'setup', callback);
         this._worker.send({
             id,
@@ -108,7 +114,7 @@ class CountWorkerObj {
         if (!this.clientConnected || !this._worker.isConnected()) {
             return callback();
         }
-        const id = uuid.v4();
+        const id = uuid();
         this._addCallback(id, 'teardown', callback);
         return this._worker.send({
             id,
@@ -118,7 +124,7 @@ class CountWorkerObj {
     }
 
     count(bucketInfo, callback) {
-        const id = uuid.v4();
+        const id = uuid();
         this._addCallback(id, 'count', (err, results) => {
             if (err) {
                 return callback(err);

CountItems/utils/createWorkers.js

@@ -2,7 +2,9 @@ const cluster = require('cluster');
 const CountWorkerObj = require('../CountWorkerObj');
 
 function createWorkers(numWorkers) {
-    if (!cluster.isMaster) return {};
+    if (!cluster.isMaster) {
+        return {};
+    }
     const workers = {};
     for (let i = 0; i < numWorkers; ++i) {
         const worker = cluster.fork(process.env);

Dockerfile

@@ -1,4 +1,4 @@
-ARG NODE_VERSION=16.20.2-bullseye-slim
+ARG NODE_VERSION=22.15.0-bookworm-slim
 
 # Use separate builder to retrieve & build node modules
 FROM node:${NODE_VERSION} AS builder
@@ -34,6 +34,7 @@ RUN apt-get update && \
         jq \
         python3 \
         python3-pip \
+        python3-venv \
     && rm -rf /var/lib/apt/lists/*
 
 ENV BALLOT_VERSION 1.0.4
@@ -48,6 +49,8 @@ COPY --from=builder /usr/src/app/supervisord /usr/local/bin/
 ENV NO_PROXY localhost,127.0.0.1
 ENV no_proxy localhost,127.0.0.1
 
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
 RUN pip install -r utapi/requirements.txt
 
 ## This section duplicates S3C Federation Dockerfile, this needs to be refactored

StalledRetry/StalledRequestHandler.js

@@ -17,7 +17,9 @@ class StalledRequestHandler {
     }
 
     queueSetup() {
-        this._queue = async.queue(({ bucket, batch, getNext }, done) => {
+        this._queue = async.queue((task, done) => {
+            const { bucket, batch, getNext } = task;
+
             if (this.queueError !== null) {
                 return done();
             }
@@ -42,15 +44,16 @@ class StalledRequestHandler {
             });
         }, this.concurrentRequests);
 
-        this._queue.error = (err, { bucket, batch }) => {
+        this._queue.error((err, task) => {
+            const { bucket, batch } = task;
             this.log.error('error occurred while processing request', {
                 error: err,
                 lastBatch: batch,
                 bucket,
             });
             this.queueError = err;
             this.kill();
-        };
+        });
     }
 
     isInProgress() {
@@ -68,7 +71,7 @@ class StalledRequestHandler {
 
     _waitForCompletion(cb) {
         async.whilst(
-            () => (!this.killed && this.isInProgress()),
+            async () => (!this.killed && this.isInProgress()),
             done => setTimeout(done, 1000),
             cb,
         );
@@ -110,7 +113,7 @@ class StalledRequestHandler {
 
         return async.times(
             this.concurrentRequests,
-            (_, cb) => nextBatch(cb),
+            (n, next) => nextBatch(next),
             err => {
                 if (err) {
                     this.log.error('failed to populate queue', {

VerifyReplication/index.js

@@ -1,5 +1,3 @@
-/* eslint-disable no-console */
-
 const { Logger } = require('werelogs');
 
 const { defaults, mandatoryVars } = require('./constants');

VerifyReplication/storage/s3.js

@@ -1,5 +1,3 @@
-/* eslint-disable no-console */
-
 const AWS = require('aws-sdk');
 const fs = require('fs');
 const http = require('http');

VerifyReplication/verifyReplication.js

@@ -89,7 +89,6 @@ function handlePrefixes(prefixList, cb) {
             prefix,
             listingLimit,
         };
-        // eslint-disable-next-line no-use-before-define
         return listAndCompare(params, done);
     }, cb);
 }

compareBuckets/compareBuckets.js

@@ -1,7 +1,3 @@
-/* eslint-disable max-len */
-/* eslint-disable no-console */
-/* eslint-disable comma-dangle */
-
 const async = require('async');
 
 const { listBucketMasterKeys } = require('./utils');
@@ -91,14 +87,13 @@ function compareBuckets(params, log, cb) {
                     if (srcDone || srcContents.length > 0) {
                         return process.nextTick(_done);
                     }
-
                     return listBucketMasterKeys(
                         bucketdSrcParams,
                         (err, isTruncated, marker, contents) => {
                             if (err) {
+                                log.error('Error fetching src bucket', { error: err });
                                 return _done(err);
                             }
-
                             srcContents = contents;
                             srcDone = !isTruncated;
                             bucketdSrcParams.marker = marker;
@@ -116,9 +111,9 @@ function compareBuckets(params, log, cb) {
                         bucketdDstParams,
                         (err, isTruncated, marker, contents) => {
                             if (err) {
+                                log.error('Error fetching dst bucket', { error: err });
                                 return _done(err);
                             }
-
                             dstContents = contents;
                             dstDone = !isTruncated;
                             bucketdDstParams.marker = marker;
@@ -129,6 +124,7 @@ function compareBuckets(params, log, cb) {
                 },
             }, err => {
                 if (err) {
+                    log.error('Parallel fetch error', { error: err });
                     return done(err);
                 }
 
@@ -224,15 +220,14 @@ function compareBuckets(params, log, cb) {
                 return process.nextTick(() => done(null));
             });
         },
-        () => (!srcDone || !dstDone || srcContents.length > 0 || dstContents.length > 0),
+        async () => (!srcDone || !dstDone || srcContents.length > 0 || dstContents.length > 0),
         err => {
             statusObj.dstBucketInProgress = null;
             statusObj.srcBucketInProgress = null;
             cb(err);
         }
     );
 }
-
 module.exports = {
     compareBuckets,
     compareObjectsReport,

compareBuckets/index.js

@@ -1,6 +1,4 @@
-/* eslint-disable max-len */
 /* eslint-disable no-console */
-/* eslint-disable comma-dangle */
 
 const { Logger } = require('werelogs');