diff --git a/bin/ensureServiceUser b/bin/ensureServiceUser index 7eaf5a5f2a..110ae42c47 100755 --- a/bin/ensureServiceUser +++ b/bin/ensureServiceUser @@ -164,9 +164,10 @@ class PolicyHandler extends BaseHandler { const res = await this.stsClient.send(command); accountId = res.Account; } catch (err) { - // Workaround a Vault issue on 8.3 branch - // https://scality.atlassian.net/browse/VAULT-238 - accountId = '000000000000'; + this.log.error('failed to get caller identity', { + error: errorUtils.reshapeExceptionError(err), + }); + throw err; } } diff --git a/conf/config.json b/conf/config.json index 46c590d60b..42460724fc 100644 --- a/conf/config.json +++ b/conf/config.json @@ -40,7 +40,11 @@ "writeConcern": "majority", "replicaSet": "rs0", "readPreference": "primary", - "database": "metadata" + "database": "metadata", + "authCredentials": { + "username": "root", + "password": "password" + } }, "kafka": { "topic": "backbeat-oplog", diff --git a/extensions/gc/tasks/GarbageCollectorTask.js b/extensions/gc/tasks/GarbageCollectorTask.js index 77e08cbd38..d0077c36bd 100644 --- a/extensions/gc/tasks/GarbageCollectorTask.js +++ b/extensions/gc/tasks/GarbageCollectorTask.js @@ -293,8 +293,7 @@ class GarbageCollectorTask extends BackbeatTask { if (err) { // if error occurs, do not commit offset unless the error is ObjNotFound // because it means the object has been deleted by other means and we don't need to retry - if (err.code === 'ObjNotFound' || err.code === 'NoSuchBucket' || - err.name === 'ObjNotFound' || err.name === 'NoSuchBucket') { + if (err.name === 'ObjNotFound' || err.name === 'NoSuchBucket') { return done(err, { committable: true }); } return done(err, { committable: false }); diff --git a/extensions/lifecycle/tasks/LifecycleColdStatusArchiveTask.js b/extensions/lifecycle/tasks/LifecycleColdStatusArchiveTask.js index c10800b7d6..22ab9aa695 100644 --- a/extensions/lifecycle/tasks/LifecycleColdStatusArchiveTask.js +++ b/extensions/lifecycle/tasks/LifecycleColdStatusArchiveTask.js @@ -86,7 +86,7 @@ class LifecycleColdStatusArchiveTask extends LifecycleUpdateTransitionTask { next => this._getMetadata(entry, log, (err, res) => { LifecycleMetrics.onS3Request(log, 'getMetadata', 'archive', err); if (err) { - if (err.code === 'ObjNotFound' || err.name === 'ObjNotFound') { + if (err.name === 'ObjNotFound') { log.info('object metadata not found, cleaning orphan cold object', { entry: entry.getLogInfo(), method: 'LifecycleColdStatusArchiveTask.processEntry', diff --git a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js index e8c69c98bc..7139d8724d 100644 --- a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js +++ b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js @@ -4,10 +4,10 @@ const ObjectMD = require('arsenal').models.ObjectMD; const { HeadObjectCommand, AbortMultipartUploadCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); const BackbeatTask = require('../../../lib/tasks/BackbeatTask'); -const { attachReqUids } = require('../../../lib/clients/utils'); const { LifecycleMetrics } = require('../LifecycleMetrics'); const { - DeleteObjectFromExpirationCommand + DeleteObjectFromExpirationCommand, + attachReqUids, } = require('@scality/cloudserverclient'); /** @typedef { import('../objectProcessor/LifecycleObjectProcessor.js') } LifecycleObjectProcessor */ @@ -56,8 +56,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask { // In this case, instead of logging an error, it should be logged as a debug message, // to avoid causing unnecessary concern to the customer. // TODO: BB-612 - const errStr = err.code || err.name; - const logLevel = errStr === 'InvalidBucketState' ? 'debug' : 'error'; + const logLevel = err.name === 'InvalidBucketState' ? 'debug' : 'error'; log[logLevel]('error getting metadata blob from S3', Object.assign({ method: 'LifecycleDeleteObjectTask._getMetadata', error: err.message, @@ -98,7 +97,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask { IfUnmodifiedSince: lastModified, }; const command = new HeadObjectCommand(reqParams); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); return s3Client.send(command) .then(res => { LifecycleMetrics.onS3Request(log, 'headObject', 'expiration', null); @@ -129,7 +128,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask { 'expiration:mpu', location, startTime - transitionTime); const command = new AbortMultipartUploadCommand(reqParams); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); client.send(command) .then(() => done(null)) .catch(done); @@ -173,7 +172,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask { .customizeDescription('Unable to obtain client')); } const s3Command = new DeleteObjectCommand(reqParams); - attachReqUids(s3Command, log); + attachReqUids(s3Command, log.getSerializedUids()); return s3Client.send(s3Command) .then(() => done(null)) .catch(done); @@ -279,7 +278,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask { // Only in S3C Backbeat API returns 'InvalidBucketState' error if the bucket is not versioned, // so we can skip checking object replication for non-versioned buckets. // TODO: BB-612 - if (err.code === 'InvalidBucketState' || err.name === 'InvalidBucketState') { + if (err.name === 'InvalidBucketState') { return done(); } return done(err); diff --git a/extensions/lifecycle/tasks/LifecycleTask.js b/extensions/lifecycle/tasks/LifecycleTask.js index b0957eca86..591ce63e81 100644 --- a/extensions/lifecycle/tasks/LifecycleTask.js +++ b/extensions/lifecycle/tasks/LifecycleTask.js @@ -16,9 +16,8 @@ const { HeadObjectCommand, GetBucketVersioningCommand, } = require('@aws-sdk/client-s3'); - +const { attachReqUids } = require('@scality/cloudserverclient'); const config = require('../../../lib/Config'); -const { attachReqUids } = require('../../../lib/clients/utils'); const BackbeatTask = require('../../../lib/tasks/BackbeatTask'); const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); const ReplicationAPI = require('../../replication/ReplicationAPI'); @@ -210,7 +209,7 @@ class LifecycleTask extends BackbeatTask { } const command = new ListObjectsCommand(params); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); async.waterfall([ next => this.s3target.send(command) .then(data => { @@ -475,7 +474,7 @@ class LifecycleTask extends BackbeatTask { */ _getMPUs(bucketData, bucketLCRules, params, nbRetries, log, done) { const command = new ListMultipartUploadsCommand(params); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); async.waterfall([ next => this.s3target.send(command) .then(data => { @@ -695,7 +694,7 @@ class LifecycleTask extends BackbeatTask { } const command = new ListObjectVersionsCommand(params); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); this.s3target.send(command) .then(data => { LifecycleMetrics.onS3Request(log, 'listObjectVersions', 'bucket', null); @@ -748,7 +747,7 @@ class LifecycleTask extends BackbeatTask { } const command = new GetObjectTaggingCommand(tagParams); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); return this.s3target.send(command) .then(tags => { LifecycleMetrics.onS3Request(log, 'getObjectTagging', 'bucket', null); @@ -1581,7 +1580,7 @@ class LifecycleTask extends BackbeatTask { params.VersionId = obj.VersionId; } const command = new HeadObjectCommand(params); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); return this.s3target.send(command) .then(data => { LifecycleMetrics.onS3Request(log, 'headObject', 'bucket', null); @@ -1888,7 +1887,7 @@ class LifecycleTask extends BackbeatTask { const command = new GetBucketVersioningCommand({ Bucket: bucketData.target.bucket, }); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); this.s3target.send(command) .then(data => { LifecycleMetrics.onS3Request( diff --git a/extensions/notification/NotificationConfigValidator.js b/extensions/notification/NotificationConfigValidator.js index 545afebdea..10a6a7b77c 100644 --- a/extensions/notification/NotificationConfigValidator.js +++ b/extensions/notification/NotificationConfigValidator.js @@ -91,7 +91,7 @@ function configValidator(backbeatConfig, extConfig) { } module.exports = { - NotificationConfigValidator: configValidator, + notificationConfigValidator: configValidator, authSchema, credentialsFileSchema, }; diff --git a/extensions/notification/index.js b/extensions/notification/index.js index 467dd559d0..2e553a3015 100644 --- a/extensions/notification/index.js +++ b/extensions/notification/index.js @@ -1,10 +1,10 @@ -const { NotificationConfigValidator } = require('./NotificationConfigValidator'); +const { notificationConfigValidator } = require('./NotificationConfigValidator'); const NotificationOplogPopulatorUtils = require('./NotificationOplogPopulatorUtils'); module.exports = { name: 'notification', version: '1.0.0', - configValidator: NotificationConfigValidator, + configValidator: notificationConfigValidator, queuePopulatorExtension: () => require('./NotificationQueuePopulator'), oplogPopulatorUtils: NotificationOplogPopulatorUtils, }; diff --git a/extensions/replication/tasks/CopyLocationTask.js b/extensions/replication/tasks/CopyLocationTask.js index d5f645beb5..b7d455e909 100644 --- a/extensions/replication/tasks/CopyLocationTask.js +++ b/extensions/replication/tasks/CopyLocationTask.js @@ -7,7 +7,7 @@ const { ObjectMD } = models; const BackbeatMetadataProxy = require('../../../lib/BackbeatMetadataProxy'); const BackbeatTask = require('../../../lib/tasks/BackbeatTask'); const { - CloudserverClient, + BackbeatRoutesClient, GetObjectCommand, MultipleBackendPutObjectCommand, MultipleBackendInitiateMPUCommand, @@ -83,7 +83,7 @@ class CopyLocationTask extends BackbeatTask { [transport === 'https' ? 'httpsAgent' : 'httpAgent']: this.sourceHTTPAgent, requestTimeout: TIMEOUT_MS, }; - this.backbeatClient = new CloudserverClient({ + this.backbeatClient = new BackbeatRoutesClient({ endpoint: `${transport}://${s3.host}:${s3.port}`, credentials: s3Credentials.getCredentialsProvider(), region: 'us-east-1', @@ -120,7 +120,7 @@ class CopyLocationTask extends BackbeatTask { return next(); }, next => this._getSourceMD(actionEntry, log, (err, objMD) => { - if (err && (err.code === 'ObjNotFound' || err.name === 'ObjNotFound')) { + if (err && (err.name === 'ObjNotFound')) { // The object was deleted before entry is processed, we // can safely skip this entry. return next(errors.ObjNotFound); @@ -867,7 +867,7 @@ class CopyLocationTask extends BackbeatTask { } log.info('action execution ended', actionEntry.getLogInfo()); // skip object if it was already transitioned - if (err && (err.InvalidObjectState || err.code === 'InvalidObjectState' || err.name === 'InvalidObjectState')) { + if (err && (err.InvalidObjectState || err.name === 'InvalidObjectState')) { log.info('object skipped: invalid object state', actionEntry.getLogInfo()); return { committable: true }; } diff --git a/extensions/replication/tasks/MultipleBackendTask.js b/extensions/replication/tasks/MultipleBackendTask.js index db0a815a6b..f0a427ccb9 100644 --- a/extensions/replication/tasks/MultipleBackendTask.js +++ b/extensions/replication/tasks/MultipleBackendTask.js @@ -19,8 +19,8 @@ const { MultipleBackendPutObjectTaggingCommand, MultipleBackendDeleteObjectTaggingCommand, addContentLengthMiddleware, + attachReqUids, } = require('@scality/cloudserverclient'); -const { attachReqUids } = require('../../../lib/clients/utils'); const getExtMetrics = require('../utils/getExtMetrics'); const { metricsExtension, metricsTypeQueued } = require('../constants'); @@ -80,7 +80,7 @@ class MultipleBackendTask extends ReplicateObject { const command = new GetBucketReplicationCommand({ Bucket: entry.getBucket(), }); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); return this.S3source.send(command) .then(data => { const replicationEnabled = data.ReplicationConfiguration.Rules @@ -261,10 +261,8 @@ class MultipleBackendTask extends ReplicateObject { const startReadTime = Date.now(); return this.backbeatSource.send(command, { abortSignal: abortController.signal }) - .then(data => { - return this._putMPUPart(sourceEntry, data.Body, size, - uploadId, partNumber, log, abortController, startReadTime, doneOnce); - }) + .then(data => this._putMPUPart(sourceEntry, data.Body, size, + uploadId, partNumber, log, abortController, startReadTime, doneOnce)) .catch(err => { // eslint-disable-next-line no-param-reassign err.origin = 'source'; @@ -893,8 +891,7 @@ class MultipleBackendTask extends ReplicateObject { */ _checkObjectState(sourceEntry, log, cb) { return this._getSourceMD(sourceEntry, log, (err, res) => { - if (err && (err.code === 'ObjNotFound' || err.name === 'ObjNotFound') && - !sourceEntry.getIsDeleteMarker()) { + if (err && err.name === 'ObjNotFound' && !sourceEntry.getIsDeleteMarker()) { // The source object was unexpectedly deleted, so we skip CRR // here. return cb(errors.InvalidObjectState); @@ -1124,7 +1121,7 @@ class MultipleBackendTask extends ReplicateObject { }); if (sourceEntry.getReplicationIsNFS()) { return this._checkObjectState(sourceEntry, log, err => { - if (err && err.code !== 'ObjNotFound' && err.name !== 'ObjNotFound') { + if (err && err.name !== 'ObjNotFound') { // If it is a non-versioned object, the object will not be // found. However we still want to replicate a delete // marker. @@ -1198,7 +1195,7 @@ class MultipleBackendTask extends ReplicateObject { return async.waterfall([ next => this._setupClients(sourceEntry, log, next), next => this._refreshSourceEntry(sourceEntry, log, (err, res) => { - if (err && (err.code === 'ObjNotFound' || err.name === 'ObjNotFound') && + if (err && err.name === 'ObjNotFound' && sourceEntry.getReplicationIsNFS() && !sourceEntry.getIsDeleteMarker()) { // The object was deleted before entry is processed, we // can safely skip this entry. @@ -1282,9 +1279,9 @@ class MultipleBackendTask extends ReplicateObject { } if (err.BadRole || err.name === 'BadRole' || (err.origin === 'source' && - (err.NoSuchEntity || err.code === 'NoSuchEntity' || err.name === 'NoSuchEntity' || - err.AccessDenied || err.code === 'AccessDenied' || err.name === 'AccessDenied' || - err.InvalidAccessKeyId || err.code === 'InvalidAccessKeyId' || err.name === 'InvalidAccessKeyId'))) { + (err.NoSuchEntity || err.name === 'NoSuchEntity' || + err.AccessDenied || err.name === 'AccessDenied' || + err.InvalidAccessKeyId || err.name === 'InvalidAccessKeyId'))) { log.error('replication failed permanently for object, ' + 'processing skipped', { failMethod: err.method, @@ -1293,13 +1290,13 @@ class MultipleBackendTask extends ReplicateObject { error: err.description }); return done(); } - if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound') { + if (err.ObjNotFound || err.name === 'ObjNotFound') { log.info('replication skipped: ' + 'source object version does not exist', { entry: sourceEntry.getLogInfo() }); return done(); } - if (err.InvalidObjectState || err.code === 'InvalidObjectState' || err.name === 'InvalidObjectState') { + if (err.InvalidObjectState || err.name === 'InvalidObjectState') { log.info('replication skipped: invalid object state', { entry: sourceEntry.getLogInfo() }); return done(); diff --git a/extensions/replication/tasks/ReplicateObject.js b/extensions/replication/tasks/ReplicateObject.js index a879c26747..02175d3c27 100644 --- a/extensions/replication/tasks/ReplicateObject.js +++ b/extensions/replication/tasks/ReplicateObject.js @@ -8,16 +8,17 @@ const ObjectMDLocation = require('arsenal').models.ObjectMDLocation; const ClientManager = require('../../../lib/clients/ClientManager'); const BackbeatMetadataProxy = require('../../../lib/BackbeatMetadataProxy'); const { - CloudserverClient, + BackbeatRoutesClient, PutDataCommand, BatchDeleteCommand, PutMetadataCommand, GetMetadataCommand, addContentLengthMiddleware, + attachReqUids, } = require('@scality/cloudserverclient'); const mapLimitWaitPendingIfError = require('../../../lib/util/mapLimitWaitPendingIfError'); -const { attachReqUids, isRetryableMiddleware, TIMEOUT_MS } = require('../../../lib/clients/utils'); +const { isRetryableMiddleware, TIMEOUT_MS } = require('../../../lib/clients/utils'); const getExtMetrics = require('../utils/getExtMetrics'); const BackbeatTask = require('../../../lib/tasks/BackbeatTask'); const { getAccountCredentials } = require('../../../lib/credentials/AccountCredentials'); @@ -128,8 +129,8 @@ class ReplicateObject extends BackbeatTask { // this call uses our own Vault client which does not set // the 'retryable' field shouldRetryFunc: err => - (err.InternalError || err.code === 'InternalError' || err.name === 'InternalError' || - err.ServiceUnavailable || err.code === 'ServiceUnavailable' || err.name === 'ServiceUnavailable'), + (err.InternalError || err.name === 'InternalError' || + err.ServiceUnavailable || err.name === 'ServiceUnavailable'), onRetryFunc: () => { this.destHosts.pickNextHost(); this._setupDestClients(this.targetRole, log); @@ -247,7 +248,7 @@ class ReplicateObject extends BackbeatTask { const command = new GetBucketReplicationCommand( { Bucket: entry.getBucket() }); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); return this.S3source.send(command) .then(data => { const replicationEnabled = ( @@ -364,6 +365,7 @@ class ReplicateObject extends BackbeatTask { Bucket: sourceEntry.getBucket(), Key: sourceEntry.getObjectKey(), VersionId: sourceEntry.getEncodedVersionId(), + RequestUids: log.getSerializedUids(), }; return this.backbeatSource.send(new GetMetadataCommand(params)) .then(data => { @@ -498,7 +500,7 @@ class ReplicateObject extends BackbeatTask { VersionId: sourceEntry.getEncodedVersionId(), PartNumber: partNumber, }); - attachReqUids(command, log); + attachReqUids(command, log.getSerializedUids()); const readStartTime = Date.now(); this.S3source.send(command, { abortSignal: abortController.signal }) @@ -656,7 +658,7 @@ class ReplicateObject extends BackbeatTask { .catch(err => { // eslint-disable-next-line no-param-reassign err.origin = 'target'; - if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound') { + if (err.ObjNotFound || err.name === 'ObjNotFound') { return cbOnce(err); } log.error('an error occurred when putting metadata to S3', @@ -748,7 +750,7 @@ class ReplicateObject extends BackbeatTask { requestTimeout: TIMEOUT_MS, connectionTimeout: TIMEOUT_MS, }; - this.backbeatSource = new CloudserverClient({ + this.backbeatSource = new BackbeatRoutesClient({ endpoint: `${this.sourceConfig.transport}://` + `${sourceS3.host}:${sourceS3.port}`, credentials: this.s3sourceCredentials.getCredentialsProvider(), @@ -803,7 +805,7 @@ class ReplicateObject extends BackbeatTask { [this.destConfig.transport === 'https' ? 'httpsAgent' : 'httpAgent']: this.destHTTPAgent, requestTimeout: TIMEOUT_MS, }; - this.backbeatDest = new CloudserverClient({ + this.backbeatDest = new BackbeatRoutesClient({ endpoint: `${this.destConfig.transport}://` + `${this.destBackbeatHost.host}:${this.destBackbeatHost.port}`, credentials: this.s3destCredentials.getCredentialsProvider(), @@ -921,8 +923,8 @@ class ReplicateObject extends BackbeatTask { } if (err.BadRole || err.name === 'BadRole' || (err.origin === 'source' && - (err.NoSuchEntity || err.code === 'NoSuchEntity' || err.name === 'NoSuchEntity' || - err.AccessDenied || err.code === 'AccessDenied' || err.name === 'AccessDenied'))) { + (err.NoSuchEntity || err.name === 'NoSuchEntity' || + err.AccessDenied || err.name === 'AccessDenied'))) { log.error('replication failed permanently for object, ' + 'processing skipped', { @@ -939,7 +941,7 @@ class ReplicateObject extends BackbeatTask { { entry: sourceEntry.getLogInfo() }); return done(); } - if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound') { + if (err.ObjNotFound || err.name === 'ObjNotFound') { if (err.origin === 'source') { log.info('replication skipped: ' + 'source object version does not exist', @@ -953,7 +955,7 @@ class ReplicateObject extends BackbeatTask { return this._processQueueEntryRetryFull( sourceEntry, destEntry, kafkaEntry, log, done); } - if (err.InvalidObjectState || err.code === 'InvalidObjectState' || err.name === 'InvalidObjectState') { + if (err.InvalidObjectState || err.name === 'InvalidObjectState') { log.info('replication skipped: invalid object state', { entry: sourceEntry.getLogInfo() }); return done(); diff --git a/extensions/utils/VaultClientWrapper.js b/extensions/utils/VaultClientWrapper.js index 5bca807d78..bb6e9ac045 100644 --- a/extensions/utils/VaultClientWrapper.js +++ b/extensions/utils/VaultClientWrapper.js @@ -1,5 +1,6 @@ const { fromTemporaryCredentials } = require('@aws-sdk/credential-providers'); const { errorUtils } = require('arsenal'); +const { GetCallerIdentityCommand } = require('@aws-sdk/client-sts'); const { authTypeAssumeRole, authTypeNone } = require('../../lib/constants'); const VaultClientCache = require('../../lib/clients/VaultClientCache'); @@ -44,10 +45,10 @@ class VaultClientWrapper { } const stsWithCreds = CredentialsManager.resolveExternalFileSync(sts, this.logger); + const endpoint = `${sts.transport || 'https'}://${sts.host}:${sts.port}`; - // FIXME: works with vault 7.10 but not 8.3 (return 501) - // https://scality.atlassian.net/browse/VAULT-238 - this._tempCredsPromise = Promise.resolve({ Account: '000000000000' }) + const getCallerIdentity = new GetCallerIdentityCommand({}); + this._tempCredsPromise = stsWithCreds.send(getCallerIdentity) .then(res => { const roleArn = `arn:aws:iam::${res.Account}:role/${roleName}`; const roleSessionName = `${this._clientId}`; @@ -57,50 +58,43 @@ class VaultClientWrapper { secretAccessKey: stsWithCreds.secretKey, }; - // Create a credential provider that assumes the role - return fromTemporaryCredentials({ - masterCredentials, + const creds = fromTemporaryCredentials({ params: { RoleArn: roleArn, RoleSessionName: roleSessionName, - // default expiration: 1 hour }, clientConfig: { - endpoint: `${this._transport}://${sts.host}:${sts.port}`, - region: 'us-east-1', - tls: this._transport === 'https', - maxAttempts: 1, - requestHandler: { - httpAgent: this._transport === 'http' ? this.stsAgent : undefined, - httpsAgent: this._transport === 'https' ? this.stsAgent : undefined, - connectionTimeout: 0, - socketTimeout: 0, - }, + endpoint, + region: sts.region, + credentials: masterCredentials, + requestHandler: this.stsAgent, }, }); + return creds(); }) - .then(creds => { - this._tempCredsPromiseResolved = true; - return creds; + .then(res => { + this._tempCreds = { + accessKey: res.accessKeyId, + secretKey: res.secretAccessKey, + sessionToken: res.sessionToken, + }; }) .catch(err => { - if (err.retryable) { - const retryDelayMs = 5000; - - this.logger.error('could not set up temporary credentials, retrying', { - retryDelayMs, - error: errorUtils.reshapeExceptionError(err), - }); - - setTimeout(() => this._storeAWSCredentialsPromise(), retryDelayMs); - } else { - this.logger.error('could not set up temporary credentials', { - error: errorUtils.reshapeExceptionError(err), - }); - } + this.logger.error('failed to get temporary credentials', { + error: errorUtils.reshapeExceptionError(err), + }); + throw err; }); } + getSTSCredentials() { + if (this._authConfig.type !== authTypeAssumeRole) { + return null; + } + + return this._tempCreds; + } + getAccountId(canonicalId, cb) { this.getAccountIds([canonicalId], (err, res) => { if (err) { diff --git a/lib/BackbeatMetadataProxy.js b/lib/BackbeatMetadataProxy.js index d2e19b2f6e..b6fc7c4816 100644 --- a/lib/BackbeatMetadataProxy.js +++ b/lib/BackbeatMetadataProxy.js @@ -106,7 +106,7 @@ class BackbeatMetadataProxy extends BackbeatTask { .catch(err => { // eslint-disable-next-line no-param-reassign err.origin = 'source'; - if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound') { + if (err.ObjNotFound || err.name === 'ObjNotFound') { return cb(err); } log.error('an error occurred when putting metadata to S3', @@ -155,7 +155,7 @@ class BackbeatMetadataProxy extends BackbeatTask { .catch(err => { // eslint-disable-next-line no-param-reassign err.origin = 'source'; - if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound') { + if (err.ObjNotFound || err.name === 'ObjNotFound') { return cbOnce(err); } log.error( @@ -165,7 +165,7 @@ class BackbeatMetadataProxy extends BackbeatTask { endpoint: this._s3Endpoint, error: err, errMsg: err.message, - errCode: err.code, + errCode: err.name, errStack: err.stack, }); return cbOnce(err); @@ -221,8 +221,7 @@ class BackbeatMetadataProxy extends BackbeatTask { // In this case, instead of logging an error, it should be logged as a debug message, // to avoid causing unnecessary concern to the customer. // TODO: BB-612 - if (err.ObjNotFound || err.code === 'ObjNotFound' || err.name === 'ObjNotFound' || - err.code === 'InvalidBucketState' || err.name === 'InvalidBucketState') { + if (err.ObjNotFound || err.name === 'ObjNotFound' || err.name === 'InvalidBucketState') { return cbOnce(err); } log.error('an error occurred when getting metadata from S3', { @@ -232,7 +231,7 @@ class BackbeatMetadataProxy extends BackbeatTask { endpoint: this._s3Endpoint, error: err, errMsg: err.message, - errCode: err.code, + errCode: err.name, errStack: err.stack, }); return cbOnce(err); @@ -240,6 +239,10 @@ class BackbeatMetadataProxy extends BackbeatTask { } listLifecycle(listType, params, log, cb) { + if (!params.RequestUids) { + // eslint-disable-next-line no-param-reassign + params.RequestUids = log.getSerializedUids(); + } if (listType === CURRENT_TYPE) { const command = new ListLifecycleCurrentsCommand(params); return this.backbeatSource.send(command) @@ -280,6 +283,7 @@ class BackbeatMetadataProxy extends BackbeatTask { getBucketIndexes(bucket, log, cb) { const command = new GetBucketIndexesCommand({ Bucket: bucket, + RequestUids: log.getSerializedUids(), }); return this.backbeatSource.send(command) @@ -302,6 +306,7 @@ class BackbeatMetadataProxy extends BackbeatTask { const command = new DeleteBucketIndexesCommand({ Bucket: bucket, Body: JSON.stringify(indexes), + RequestUids: log.getSerializedUids(), }); return this.backbeatSource.send(command) @@ -344,7 +349,7 @@ class BackbeatMetadataProxy extends BackbeatTask { connectionTimeout: TIMEOUT_MS }; const creds = this._createCredentials(log); - this.backbeatSource = new CloudserverClient({ + this.backbeatSource = new BackbeatRoutesClient({ endpoint: this._s3Endpoint, credentials: creds.getCredentialsProvider(), region: 'us-east-1', diff --git a/lib/clients/ClientManager.js b/lib/clients/ClientManager.js index cebba2afe1..5468fa17e0 100644 --- a/lib/clients/ClientManager.js +++ b/lib/clients/ClientManager.js @@ -3,9 +3,7 @@ const BackbeatMetadataProxy = require('../BackbeatMetadataProxy'); const { createS3Client, isRetryableMiddleware, TIMEOUT_MS } = require('./utils'); const { authTypeAssumeRole } = require('../constants'); const { http: HttpAgent, https: HttpsAgent } = require('httpagent'); -const { - CloudserverClient, -} = require('@scality/cloudserverclient'); +const { BackbeatRoutesClient } = require('@scality/cloudserverclient'); // TODO: test inactive credential deletion const DELETE_INACTIVE_CREDENTIALS_INTERVAL = 1000 * 60 * 30; // 30m @@ -153,7 +151,7 @@ class ClientManager { [this._transport === 'https' ? 'httpsAgent' : 'httpAgent']: this.s3Agent, requestTimeout: TIMEOUT_MS, }; - this.backbeatClients[accountId] = new CloudserverClient({ + this.backbeatClients[accountId] = new BackbeatRoutesClient({ endpoint: `${this._transport}://${this._s3Config.host}:${this._s3Config.port}`, credentials: credentials.getCredentialsProvider(), region: 'us-east-1', diff --git a/lib/clients/utils.js b/lib/clients/utils.js index 56653136a1..0b0cf4b9af 100644 --- a/lib/clients/utils.js +++ b/lib/clients/utils.js @@ -9,22 +9,6 @@ const { const TIMEOUT_MS = 1000 * 60 * 2; // 2 minutes in ms -function attachReqUids(s3req, log) { - s3req.middlewareStack.add( - next => async args => { - if (args.request && args.request.headers) { - // eslint-disable-next-line no-param-reassign - args.request.headers['X-Scal-Request-Uids'] = log.getSerializedUids(); - } - return next(args); - }, - { - step: 'build', - name: 'attachReqUids', - } - ); -} - function isRetryableMiddleware() { return next => async args => { try { @@ -89,7 +73,6 @@ function createS3Client(params) { } module.exports = { - attachReqUids, createS3Client, isRetryableMiddleware, TIMEOUT_MS, diff --git a/lib/queuePopulator/IngestionProducer.js b/lib/queuePopulator/IngestionProducer.js index bfdcf6bab6..c3ce2575df 100644 --- a/lib/queuePopulator/IngestionProducer.js +++ b/lib/queuePopulator/IngestionProducer.js @@ -9,17 +9,18 @@ const ObjectMD = require('arsenal').models.ObjectMD; const VID_SEP = require('arsenal').versioning.VersioningConstants .VersionId.Separator; -const { attachReqUids, isRetryableMiddleware } = require('../clients/utils'); +const { isRetryableMiddleware } = require('../clients/utils'); const RaftLogEntry = require('../models/RaftLogEntry'); const IngestionPopulatorMetrics = require('./IngestionPopulatorMetrics'); const { http: HttpAgent, https: HttpsAgent } = require('httpagent'); const { - CloudserverClient, + BackbeatRoutesClient, GetRaftIdCommand, GetRaftLogCommand, GetRaftBucketsCommand, GetBucketCseqCommand, - GetMetadataCommand + GetMetadataCommand, + attachReqUids, } = require('@scality/cloudserverclient'); class ListRecordStream extends stream.Transform { @@ -95,7 +96,7 @@ class IngestionProducer { requestTimeout: 0, }; - this._ringReader = new CloudserverClient({ + this._ringReader = new BackbeatRoutesClient({ endpoint, credentials, region: 'us-east-1', @@ -342,7 +343,7 @@ class IngestionProducer { } // TODO: For testing, I can set MaxKeys const command = new ListObjectVersionsCommand(params); - attachReqUids(command, this.requestLogger); + attachReqUids(command, this.requestLogger.getSerializedUids()); return this._s3Client.send(command) .then(data => { const { @@ -490,7 +491,8 @@ class IngestionProducer { */ _getBucketCseq(bucket, done) { const command = new GetBucketCseqCommand({ - Bucket: bucket + Bucket: bucket, + RequestUids: this.requestLogger.getSerializedUids(), }); return this._ringReader.send(command) diff --git a/lib/tasks/BackbeatTask.js b/lib/tasks/BackbeatTask.js index a6db3e1291..873d8fba9f 100644 --- a/lib/tasks/BackbeatTask.js +++ b/lib/tasks/BackbeatTask.js @@ -62,12 +62,18 @@ class BackbeatTask { const _handleRes = (...args) => { const err = args[0]; if (err) { - if (err.retryable === undefined && - (err.code === 'ECONNRESET' || err.code === 'EPIPE' || - err.code === 'ETIMEDOUT')) { + if (err.retryable === undefined) { // Network/socket errors are actually retryable, though they do not get flagged // as such by AWS-SDK client - err.retryable = true; + if (err.code === 'ECONNRESET' || err.code === 'EPIPE' || + err.code === 'ETIMEDOUT') { + err.retryable = true; + } else if (err.name === 'TimeoutError' || + err.message?.includes('ECONNRESET') || + err.message.includes('EPIPE')) { + // sdk v3 errors + err.retryable = true; + } } if (!shouldRetryFunc(err)) { diff --git a/package.json b/package.json index 5b5726dce2..d196436b55 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "@aws-sdk/client-s3": "^3.921.0", "@aws-sdk/client-sts": "^3.921.0", "@aws-sdk/credential-providers": "^3.921.0", - "@scality/cloudserverclient": "git+https://github.com/scality/cloudserverclient#1.0.0", + "@scality/cloudserverclient": "^1.0.2", "@smithy/node-http-handler": "^3.3.3", "JSONStream": "^1.3.5", "arsenal": "git+https://github.com/scality/arsenal#8.3.0-preview.1", diff --git a/tests/functional/ingestion/S3Mock.js b/tests/functional/ingestion/S3Mock.js index 68264ce532..8c76797d48 100644 --- a/tests/functional/ingestion/S3Mock.js +++ b/tests/functional/ingestion/S3Mock.js @@ -8,12 +8,12 @@ const { DeleteBucketCommand, } = require('@aws-sdk/client-s3'); -const { CloudserverClient, GetObjectListCommand } = require('@scality/cloudserverclient'); +const { BackbeatRoutesClient, GetObjectListCommand } = require('@scality/cloudserverclient'); function getClients(sourceInfo) { const { port } = sourceInfo; - const backbeatClient = new CloudserverClient({ + const backbeatClient = new BackbeatRoutesClient({ endpoint: `http://localhost:${port}`, credentials: { accessKeyId: 'accessKey1', diff --git a/tests/functional/lib/BackbeatClient.js b/tests/functional/lib/BackbeatClient.js index 6f25f62180..6b7b5efcdc 100644 --- a/tests/functional/lib/BackbeatClient.js +++ b/tests/functional/lib/BackbeatClient.js @@ -6,7 +6,7 @@ const { getAccountCredentials } = const { MetadataMock, mockLogs, objectList, dummyBucketMD, objectMD } = require('../utils/MetadataMock'); const { - CloudserverClient, + BackbeatRoutesClient, GetRaftIdCommand, GetRaftBucketsCommand, GetRaftLogCommand, @@ -30,7 +30,7 @@ const accountCreds = getAccountCredentials({ account: 'bart', }); -const backbeatClient = new CloudserverClient({ +const backbeatClient = new BackbeatRoutesClient({ endpoint: `http://localhost:${backbeatClientTestPort}`, credentials: accountCreds.getCredentialsProvider(), region: 'us-east-1', diff --git a/tests/unit/notification/NotificationConfigValidator.js b/tests/unit/notification/NotificationConfigValidator.js index fee1d85405..cbc6db1a15 100644 --- a/tests/unit/notification/NotificationConfigValidator.js +++ b/tests/unit/notification/NotificationConfigValidator.js @@ -1,6 +1,6 @@ const assert = require('assert'); -const { NotificationConfigValidator } = require('../../../extensions/notification/NotificationConfigValidator'); +const { notificationConfigValidator } = require('../../../extensions/notification/NotificationConfigValidator'); const defaultExtConfig = { topic: 'topic', @@ -314,7 +314,7 @@ describe('NotificationConfigValidator ::', () => { destinations: [testCase.destinationConfig], }; const tester = testCase.valid ? assert.doesNotThrow : assert.throws; - tester(() => NotificationConfigValidator(null, extConfig)); + tester(() => notificationConfigValidator(null, extConfig)); }) ); }); diff --git a/yarn.lock b/yarn.lock index 4d25bba4de..26a369fbb7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2129,9 +2129,10 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@scality/cloudserverclient@git+https://github.com/scality/cloudserverclient#1.0.0": - version "1.0.0" - resolved "git+https://github.com/scality/cloudserverclient#5be9d7bb697ac7010ed1a0f862748c0b4840aa71" +"@scality/cloudserverclient@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@scality/cloudserverclient/-/cloudserverclient-1.0.2.tgz#8d30aafc7dde7ad09d4dcac7323eb2161ce98ffe" + integrity sha512-7Q4b9LQxUGB6DqNxZPDT6rxUAhWgJlXVAOy9ZHY7Vu4m2UOfvbaaxBXo7C4T3d2lxwdgOEWbdc+4GZoykqRI9Q== dependencies: "@aws-sdk/client-s3" "^3.896.0" JSONStream "^1.3.5"