Skip to content

sdk v3 migration : follow up on a few details #2703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
SylvainSenechal wants to merge 7 commits into
base: development/9.2
Choose a base branch
from
Draft

sdk v3 migration : follow up on a few details #2703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4bd3589
pull cloudserverclient from npm
SylvainSenechal Jan 16, 2026
d6a1d5d
remove todo from vault-238
SylvainSenechal Jan 19, 2026
d2c51dd
remove err.code checks following sdk v3 migration
SylvainSenechal Jan 20, 2026
90f18a1
fix linter warnings
SylvainSenechal Jan 20, 2026
abed00a
refactor attachRequestUids
SylvainSenechal Jan 20, 2026
07ad4e0
rename cloudserver client
SylvainSenechal Jan 20, 2026
47f9e10
bump cloudserver client to 1.0.2
SylvainSenechal Jan 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions bin/ensureServiceUser
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,10 @@ class PolicyHandler extends BaseHandler {
const res = await this.stsClient.send(command);
accountId = res.Account;
} catch (err) {
// Workaround a Vault issue on 8.3 branch
Copy link
Contributor Author

@SylvainSenechal SylvainSenechal Jan 21, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://scality.atlassian.net/browse/VAULT-238
Get caller identity is available in vault
image

// https://scality.atlassian.net/browse/VAULT-238
accountId = '000000000000';
this.log.error('failed to get caller identity', {
error: errorUtils.reshapeExceptionError(err),
});
throw err;
}
}

Expand Down
6 changes: 5 additions & 1 deletion conf/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,11 @@
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "metadata"
"database": "metadata",
"authCredentials": {
"username": "root",
"password": "password"
}
},
"kafka": {
"topic": "backbeat-oplog",
Expand Down
3 changes: 1 addition & 2 deletions extensions/gc/tasks/GarbageCollectorTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,8 +293,7 @@ class GarbageCollectorTask extends BackbeatTask {
if (err) {
// if error occurs, do not commit offset unless the error is ObjNotFound
// because it means the object has been deleted by other means and we don't need to retry
if (err.code === 'ObjNotFound' || err.code === 'NoSuchBucket' ||
err.name === 'ObjNotFound' || err.name === 'NoSuchBucket') {
if (err.name === 'ObjNotFound' || err.name === 'NoSuchBucket') {
return done(err, { committable: true });
}
return done(err, { committable: false });
Expand Down
2 changes: 1 addition & 1 deletion extensions/lifecycle/tasks/LifecycleColdStatusArchiveTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ class LifecycleColdStatusArchiveTask extends LifecycleUpdateTransitionTask {
next => this._getMetadata(entry, log, (err, res) => {
LifecycleMetrics.onS3Request(log, 'getMetadata', 'archive', err);
if (err) {
if (err.code === 'ObjNotFound' || err.name === 'ObjNotFound') {
if (err.name === 'ObjNotFound') {
log.info('object metadata not found, cleaning orphan cold object', {
entry: entry.getLogInfo(),
method: 'LifecycleColdStatusArchiveTask.processEntry',
Expand Down
15 changes: 7 additions & 8 deletions extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ const ObjectMD = require('arsenal').models.ObjectMD;
const { HeadObjectCommand, AbortMultipartUploadCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');

const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
const { attachReqUids } = require('../../../lib/clients/utils');
const { LifecycleMetrics } = require('../LifecycleMetrics');
const {
DeleteObjectFromExpirationCommand
DeleteObjectFromExpirationCommand,
attachReqUids,
} = require('@scality/cloudserverclient');
/** @typedef { import('../objectProcessor/LifecycleObjectProcessor.js') } LifecycleObjectProcessor */

Expand Down Expand Up @@ -56,8 +56,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
// In this case, instead of logging an error, it should be logged as a debug message,
// to avoid causing unnecessary concern to the customer.
// TODO: BB-612
const errStr = err.code || err.name;
const logLevel = errStr === 'InvalidBucketState' ? 'debug' : 'error';
const logLevel = err.name === 'InvalidBucketState' ? 'debug' : 'error';
log[logLevel]('error getting metadata blob from S3', Object.assign({
method: 'LifecycleDeleteObjectTask._getMetadata',
error: err.message,
Expand Down Expand Up @@ -98,7 +97,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
IfUnmodifiedSince: lastModified,
};
const command = new HeadObjectCommand(reqParams);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
return s3Client.send(command)
.then(res => {
LifecycleMetrics.onS3Request(log, 'headObject', 'expiration', null);
Expand Down Expand Up @@ -129,7 +128,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
'expiration:mpu',
location, startTime - transitionTime);
const command = new AbortMultipartUploadCommand(reqParams);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
client.send(command)
.then(() => done(null))
.catch(done);
Expand Down Expand Up @@ -173,7 +172,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
.customizeDescription('Unable to obtain client'));
}
const s3Command = new DeleteObjectCommand(reqParams);
attachReqUids(s3Command, log);
attachReqUids(s3Command, log.getSerializedUids());
return s3Client.send(s3Command)
.then(() => done(null))
.catch(done);
Expand Down Expand Up @@ -279,7 +278,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
// <!> Only in S3C <!> Backbeat API returns 'InvalidBucketState' error if the bucket is not versioned,
// so we can skip checking object replication for non-versioned buckets.
// TODO: BB-612
if (err.code === 'InvalidBucketState' || err.name === 'InvalidBucketState') {
if (err.name === 'InvalidBucketState') {
return done();
}
return done(err);
Expand Down
15 changes: 7 additions & 8 deletions extensions/lifecycle/tasks/LifecycleTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ const {
HeadObjectCommand,
GetBucketVersioningCommand,
} = require('@aws-sdk/client-s3');

const { attachReqUids } = require('@scality/cloudserverclient');
const config = require('../../../lib/Config');
const { attachReqUids } = require('../../../lib/clients/utils');
const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry');
const ReplicationAPI = require('../../replication/ReplicationAPI');
Expand Down Expand Up @@ -210,7 +209,7 @@ class LifecycleTask extends BackbeatTask {
}

const command = new ListObjectsCommand(params);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
async.waterfall([
next => this.s3target.send(command)
.then(data => {
Expand Down Expand Up @@ -475,7 +474,7 @@ class LifecycleTask extends BackbeatTask {
*/
_getMPUs(bucketData, bucketLCRules, params, nbRetries, log, done) {
const command = new ListMultipartUploadsCommand(params);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
async.waterfall([
next => this.s3target.send(command)
.then(data => {
Expand Down Expand Up @@ -695,7 +694,7 @@ class LifecycleTask extends BackbeatTask {
}

const command = new ListObjectVersionsCommand(params);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
this.s3target.send(command)
.then(data => {
LifecycleMetrics.onS3Request(log, 'listObjectVersions', 'bucket', null);
Expand Down Expand Up @@ -748,7 +747,7 @@ class LifecycleTask extends BackbeatTask {
}

const command = new GetObjectTaggingCommand(tagParams);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
return this.s3target.send(command)
.then(tags => {
LifecycleMetrics.onS3Request(log, 'getObjectTagging', 'bucket', null);
Expand Down Expand Up @@ -1581,7 +1580,7 @@ class LifecycleTask extends BackbeatTask {
params.VersionId = obj.VersionId;
}
const command = new HeadObjectCommand(params);
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
return this.s3target.send(command)
.then(data => {
LifecycleMetrics.onS3Request(log, 'headObject', 'bucket', null);
Expand Down Expand Up @@ -1888,7 +1887,7 @@ class LifecycleTask extends BackbeatTask {
const command = new GetBucketVersioningCommand({
Bucket: bucketData.target.bucket,
});
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
this.s3target.send(command)
.then(data => {
LifecycleMetrics.onS3Request(
Expand Down
2 changes: 1 addition & 1 deletion extensions/notification/NotificationConfigValidator.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ function configValidator(backbeatConfig, extConfig) {
}

module.exports = {
NotificationConfigValidator: configValidator,
notificationConfigValidator: configValidator,
authSchema,
credentialsFileSchema,
};
4 changes: 2 additions & 2 deletions extensions/notification/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
const { NotificationConfigValidator } = require('./NotificationConfigValidator');
const { notificationConfigValidator } = require('./NotificationConfigValidator');
const NotificationOplogPopulatorUtils = require('./NotificationOplogPopulatorUtils');

module.exports = {
name: 'notification',
version: '1.0.0',
configValidator: NotificationConfigValidator,
configValidator: notificationConfigValidator,
queuePopulatorExtension: () => require('./NotificationQueuePopulator'),
oplogPopulatorUtils: NotificationOplogPopulatorUtils,
};
8 changes: 4 additions & 4 deletions extensions/replication/tasks/CopyLocationTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ const { ObjectMD } = models;
const BackbeatMetadataProxy = require('../../../lib/BackbeatMetadataProxy');
const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
const {
CloudserverClient,
BackbeatRoutesClient,
GetObjectCommand,
MultipleBackendPutObjectCommand,
MultipleBackendInitiateMPUCommand,
Expand Down Expand Up @@ -83,7 +83,7 @@ class CopyLocationTask extends BackbeatTask {
[transport === 'https' ? 'httpsAgent' : 'httpAgent']: this.sourceHTTPAgent,
requestTimeout: TIMEOUT_MS,
};
this.backbeatClient = new CloudserverClient({
this.backbeatClient = new BackbeatRoutesClient({
endpoint: `${transport}://${s3.host}:${s3.port}`,
credentials: s3Credentials.getCredentialsProvider(),
region: 'us-east-1',
Expand Down Expand Up @@ -120,7 +120,7 @@ class CopyLocationTask extends BackbeatTask {
return next();
},
next => this._getSourceMD(actionEntry, log, (err, objMD) => {
if (err && (err.code === 'ObjNotFound' || err.name === 'ObjNotFound')) {
if (err && (err.name === 'ObjNotFound')) {
// The object was deleted before entry is processed, we
// can safely skip this entry.
return next(errors.ObjNotFound);
Expand Down Expand Up @@ -867,7 +867,7 @@ class CopyLocationTask extends BackbeatTask {
}
log.info('action execution ended', actionEntry.getLogInfo());
// skip object if it was already transitioned
if (err && (err.InvalidObjectState || err.code === 'InvalidObjectState' || err.name === 'InvalidObjectState')) {
if (err && (err.InvalidObjectState || err.name === 'InvalidObjectState')) {
log.info('object skipped: invalid object state', actionEntry.getLogInfo());
return { committable: true };
}
Expand Down
27 changes: 12 additions & 15 deletions extensions/replication/tasks/MultipleBackendTask.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ const {
MultipleBackendPutObjectTaggingCommand,
MultipleBackendDeleteObjectTaggingCommand,
addContentLengthMiddleware,
attachReqUids,
} = require('@scality/cloudserverclient');
const { attachReqUids } = require('../../../lib/clients/utils');
const getExtMetrics = require('../utils/getExtMetrics');
const { metricsExtension, metricsTypeQueued } = require('../constants');

Expand Down Expand Up @@ -80,7 +80,7 @@ class MultipleBackendTask extends ReplicateObject {
const command = new GetBucketReplicationCommand({
Bucket: entry.getBucket(),
});
attachReqUids(command, log);
attachReqUids(command, log.getSerializedUids());
return this.S3source.send(command)
.then(data => {
const replicationEnabled = data.ReplicationConfiguration.Rules
Expand Down Expand Up @@ -261,10 +261,8 @@ class MultipleBackendTask extends ReplicateObject {

const startReadTime = Date.now();
return this.backbeatSource.send(command, { abortSignal: abortController.signal })
.then(data => {
return this._putMPUPart(sourceEntry, data.Body, size,
uploadId, partNumber, log, abortController, startReadTime, doneOnce);
})
.then(data => this._putMPUPart(sourceEntry, data.Body, size,
uploadId, partNumber, log, abortController, startReadTime, doneOnce))
.catch(err => {
// eslint-disable-next-line no-param-reassign
err.origin = 'source';
Expand Down Expand Up @@ -893,8 +891,7 @@ class MultipleBackendTask extends ReplicateObject {
*/
_checkObjectState(sourceEntry, log, cb) {
return this._getSourceMD(sourceEntry, log, (err, res) => {
if (err && (err.code === 'ObjNotFound' || err.name === 'ObjNotFound') &&
!sourceEntry.getIsDeleteMarker()) {
if (err && err.name === 'ObjNotFound' && !sourceEntry.getIsDeleteMarker()) {
// The source object was unexpectedly deleted, so we skip CRR
// here.
return cb(errors.InvalidObjectState);
Expand Down Expand Up @@ -1124,7 +1121,7 @@ class MultipleBackendTask extends ReplicateObject {
});
if (sourceEntry.getReplicationIsNFS()) {
return this._checkObjectState(sourceEntry, log, err => {
if (err && err.code !== 'ObjNotFound' && err.name !== 'ObjNotFound') {
if (err && err.name !== 'ObjNotFound') {
// If it is a non-versioned object, the object will not be
// found. However we still want to replicate a delete
// marker.
Expand Down Expand Up @@ -1198,7 +1195,7 @@ class MultipleBackendTask extends ReplicateObject {
return async.waterfall([
next => this._setupClients(sourceEntry, log, next),
next => this._refreshSourceEntry(sourceEntry, log, (err, res) => {
if (err && (err.code === 'ObjNotFound' || err.name === 'ObjNotFound') &&
if (err && err.name === 'ObjNotFound' &&
sourceEntry.getReplicationIsNFS() && !sourceEntry.getIsDeleteMarker()) {
// The object was deleted before entry is processed, we
// can safely skip this entry.
Expand Down Expand Up @@ -1282,9 +1279,9 @@ class MultipleBackendTask extends ReplicateObject {
}
if (err.BadRole || err.name === 'BadRole' ||
(err.origin === 'source' &&
(err.NoSuchEntity || err.code === 'NoSuchEntity' || err.name === 'NoSuchEntity' ||
err.AccessDenied || err.code === 'AccessDenied' || err.name === 'AccessDenied' ||
err.InvalidAccessKeyId || err.code === 'InvalidAccessKeyId' || err.name === 'InvalidAccessKeyId'))) {
(err.NoSuchEntity || err.name === 'NoSuchEntity' ||
err.AccessDenied || err.name === 'AccessDenied' ||
err.InvalidAccessKeyId || err.name === 'InvalidAccessKeyId'))) {
log.error('replication failed permanently for object, ' +
'processing skipped',
{ failMethod: err.method,
Expand All @@ -1293,13 +1290,13 @@ class MultipleBackendTask extends ReplicateObject {
error: err.description });
return done();
}
if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound') {
if (err.ObjNotFound || err.name === 'ObjNotFound') {
log.info('replication skipped: ' +
'source object version does not exist',
{ entry: sourceEntry.getLogInfo() });
return done();
}
if (err.InvalidObjectState || err.code === 'InvalidObjectState' || err.name === 'InvalidObjectState') {
if (err.InvalidObjectState || err.name === 'InvalidObjectState') {
log.info('replication skipped: invalid object state',
{ entry: sourceEntry.getLogInfo() });
return done();
Expand Down
Loading
Loading