Skip to content

Commit a7de3ce

Browse files
SylvainSenechalSylvainSenechal
committed
update scripts with new cloudserverclient
ISSUE: S3UTILS-205
1 parent c78cde0 commit a7de3ce

File tree

6 files changed

+1364
-1636
lines changed

6 files changed

+1364
-1636
lines changed

‎CRR/ReplicationStatusUpdater.js‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ const {
22
doWhilst, eachSeries, eachLimit, waterfall,
33
} = require('async');
44
const { ObjectMD } = require('arsenal').models;
5-
const { setupClients } = require('./clients');
65
const {
76
ListObjectVersionsCommand,
87
GetBucketReplicationCommand
98
} = require('@aws-sdk/client-s3');
9+
const CloudserverClient = require('../Clients/CloudserverClient');
10+
const createS3Client = require('../Clients/s3Client');
1011

1112
const LOG_PROGRESS_INTERVAL_MS = 10000;
1213

@@ -86,19 +87,17 @@ class ReplicationStatusUpdater {
8687
}
8788

8889
/**
89-
* Sets up and initializes the S3 and Backbeat client instances.
90+
* Sets up and initializes the S3 and Cloudserver client instances.
9091
*
91-
* @returns {void} This method does not return a value; instead, it sets the S3 and Backbeat clients.
92+
* @returns {void} This method does not return a value; instead, it sets the S3 and Cloudserver clients.
9293
*/
9394
_setupClients() {
94-
const { s3, bb } = setupClients({
95+
this.s3 = createS3Client({
9596
accessKey: this.accessKey,
9697
secretKey: this.secretKey,
9798
endpoint: this.endpoint,
9899
}, this.log);
99-
100-
this.s3 = s3;
101-
this.bb = bb;
100+
this.cloudserverclient = new CloudserverClient(this.endpoint, this.accessKey, this.secretKey);
102101
}
103102

104103
/**
@@ -161,7 +160,7 @@ class ReplicationStatusUpdater {
161160
let skip = false;
162161
return waterfall([
163162
// get object blob
164-
next => this.bb.getMetadata({
163+
next => this.cloudserverclient.getMetadata({
165164
Bucket: bucket,
166165
Key: key,
167166
VersionId: versionId,
@@ -245,11 +244,10 @@ class ReplicationStatusUpdater {
245244
objMD.setReplicationStatus('PENDING');
246245
objMD.updateMicroVersionId();
247246
const md = objMD.getSerialized();
248-
return this.bb.putMetadata({
247+
return this.cloudserverclient.putMetadata({
249248
Bucket: bucket,
250249
Key: key,
251250
VersionId: versionId,
252-
ContentLength: Buffer.byteLength(md),
253251
Body: md,
254252
}, next);
255253
},

‎cleanupNoncurrentVersions.js‎

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const { doWhilst, eachSeries, filterLimit } = require('async');
1010
const { Logger } = require('werelogs');
1111

1212
const BackbeatClient = require('./BackbeatClient');
13+
const CloudserverClient = require('./Clients/CloudserverClient');
1314
const parseOlderThan = require('./utils/parseOlderThan');
1415

1516
const log = new Logger('s3utils::cleanupNoncurrentVersions');
@@ -199,41 +200,12 @@ const s3 = new S3Client({
199200
),
200201
});
201202

202-
const options = {
203-
accessKeyId: ACCESS_KEY,
204-
secretAccessKey: SECRET_KEY,
205-
endpoint: S3_ENDPOINT,
206-
region: 'us-east-1',
207-
sslEnabled: s3EndpointIsHttps,
208-
s3ForcePathStyle: true,
209-
apiVersions: { s3: '2006-03-01' },
210-
signatureVersion: 'v4',
211-
signatureCache: false,
212-
httpOptions: {
213-
timeout: 0,
214-
agent,
215-
},
216-
};
217-
/**
218-
* Options specific to s3 requests
219-
* `maxRetries` & `customBackoff` are set only to s3 requests
220-
* default aws sdk retry count is 3 with an exponential delay of 2^n * 30 ms
221-
*/
222-
const s3Options = {
223-
maxRetries: AWS_SDK_REQUEST_RETRIES,
224-
customBackoff: (retryCount, error) => {
225-
log.error('aws sdk request error', { error, retryCount });
226-
// retry with exponential backoff delay capped at 1mn max
227-
// between retries, and a little added jitter
228-
return Math.min(AWS_SDK_REQUEST_INITIAL_DELAY_MS
229-
* 2 ** retryCount, 60000)
230-
* (0.9 + Math.random() * 0.2);
231-
},
232-
};
233-
234-
const opt = Object.assign(options, s3Options);
235-
236-
const bb = new BackbeatClient(opt);
203+
const cloudserverclient = new CloudserverClient(
204+
S3_ENDPOINT,
205+
ACCESS_KEY,
206+
SECRET_KEY,
207+
agent,
208+
);
237209

238210
let nListed = 0;
239211
let nDeletesTriggered = 0;
@@ -283,7 +255,7 @@ function _listObjectVersions(bucket, VersionIdMarker, KeyMarker, cb) {
283255
}
284256

285257
function _getMetadata(bucket, key, versionId, cb) {
286-
return bb.getMetadata({
258+
return cloudserverclient.getMetadata({
287259
Bucket: bucket,
288260
Key: key,
289261
VersionId: versionId,

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
"JSONStream": "^1.3.5",
3434
"arsenal": "git+https://github.com/scality/arsenal#8.2.36",
3535
"async": "^3.2.6",
36-
"aws-sdk": "^2.1691.0",
3736
"bucketclient": "scality/bucketclient#8.2.6",
3837
"express": "^5.1.0",
3938
"heapdump": "^0.3.15",
@@ -47,7 +46,8 @@
4746
"vaultclient": "scality/vaultclient#8.5.1",
4847
"werelogs": "scality/werelogs#8.2.1",
4948
"xml2js": "^0.6.2",
50-
"zenkoclient": "scality/zenkoclient#1.2.1"
49+
"zenkoclient": "scality/zenkoclient#1.2.1",
50+
"@scality/cloudserverclient": "git+https://github.com/scality/cloudserverclient#v1.0.0"
5151
},
5252
"resolutions": {
5353
"ioctl": "2.0.2",

‎requeueFailedCRRCronJob.js‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ const VID_SEP = require('arsenal').versioning.VersioningConstants
1111
.VersionId.Separator;
1212
const { Logger } = require('werelogs');
1313

14-
const BackbeatClient = require('./BackbeatClient');
14+
const CloudserverClient = require('./Clients/CloudserverClient');
1515

1616
const log = new Logger('s3utils::requeueFailedCRRCronJob');
1717
const {
@@ -54,23 +54,11 @@ const PRODUCER_RETRY_DELAY_MS = 5000;
5454
const PRODUCER_MAX_RETRIES = 60;
5555
const PRODUCER_POLL_INTERVAL_MS = 2000;
5656

57-
const bbOptions = {
58-
accessKeyId: ACCESS_KEY,
59-
secretAccessKey: SECRET_KEY,
60-
endpoint: CLOUDSERVER_ENDPOINT,
61-
region: 'us-east-1',
62-
sslEnabled: false,
63-
s3ForcePathStyle: true,
64-
apiVersions: { s3: '2006-03-01' },
65-
signatureVersion: 'v4',
66-
signatureCache: false,
67-
httpOptions: {
68-
timeout: 0,
69-
agent: new httpArsn.Agent({ keepAlive: true }),
70-
},
71-
};
72-
73-
const bb = new BackbeatClient(bbOptions);
57+
const cloudserverclient = new CloudserverClient(
58+
CLOUDSERVER_ENDPOINT,
59+
ACCESS_KEY,
60+
SECRET_KEY,
61+
);
7462

7563
const producer = new Producer({
7664
'metadata.broker.list': KAFKA_HOSTS,
@@ -88,7 +76,7 @@ function _requeueObject(bucket, key, versionId, counters, cb) {
8876
}
8977
return waterfall([
9078
// get object blob
91-
next => bb.getMetadata({
79+
next => cloudserverclient.getMetadata({
9280
Bucket: bucket,
9381
Key: key,
9482
VersionId: versionId,

‎tests/unit/CRR/ReplicationStatusUpdater.js‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,14 @@ describe('ReplicationStatusUpdater', () => {
4646
})
4747
}));
4848

49-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(1);
50-
expect(crr.bb.getMetadata).toHaveBeenCalledWith({
49+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(1);
50+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledWith({
5151
Bucket: 'bucket0',
5252
Key: listVersionRes.Versions[0].Key,
5353
VersionId: listVersionRes.Versions[0].VersionId,
5454
}, expect.any(Function));
5555

56-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
56+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
5757
const expectedReplicationInfo = {
5858
status: 'PENDING',
5959
backends: [
@@ -70,7 +70,7 @@ describe('ReplicationStatusUpdater', () => {
7070
storageType: '',
7171
dataStoreVersionId: '',
7272
};
73-
expect(crr.bb.putMetadata).toHaveBeenCalledWith(
73+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledWith(
7474
expect.objectContaining({
7575
Body: expect.stringContaining(JSON.stringify(expectedReplicationInfo)),
7676
}),
@@ -119,20 +119,20 @@ describe('ReplicationStatusUpdater', () => {
119119
})
120120
}));
121121

122-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(2);
123-
expect(crr.bb.getMetadata).toHaveBeenNthCalledWith(1, {
122+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(2);
123+
expect(crr.cloudserverclient.getMetadata).toHaveBeenNthCalledWith(1, {
124124
Bucket: 'bucket0',
125125
Key: listVersionsRes.Versions[0].Key,
126126
VersionId: listVersionsRes.Versions[0].VersionId,
127127
}, expect.any(Function));
128128

129-
expect(crr.bb.getMetadata).toHaveBeenNthCalledWith(2, {
129+
expect(crr.cloudserverclient.getMetadata).toHaveBeenNthCalledWith(2, {
130130
Bucket: 'bucket0',
131131
Key: listVersionsRes.Versions[1].Key,
132132
VersionId: listVersionsRes.Versions[1].VersionId,
133133
}, expect.any(Function));
134134

135-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(2);
135+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(2);
136136

137137
assert.strictEqual(crr._nProcessed, 2);
138138
assert.strictEqual(crr._nSkipped, 0);
@@ -365,7 +365,7 @@ describe('ReplicationStatusUpdater', () => {
365365
listingLimit: 10,
366366
siteName: 'aws-location',
367367
}, logger);
368-
crr.bb.getMetadata = jest.fn((p, cb) => {
368+
crr.cloudserverclient.getMetadata = jest.fn((p, cb) => {
369369
const objectMd = JSON.parse(getMetadataRes.Body);
370370
objectMd.replicationInfo = params.replicationInfo;
371371
cb(null, { Body: JSON.stringify(objectMd) });
@@ -395,9 +395,9 @@ describe('ReplicationStatusUpdater', () => {
395395
})
396396
}));
397397

398-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(1);
399-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
400-
expect(crr.bb.putMetadata).toHaveBeenCalledWith(
398+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(1);
399+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
400+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledWith(
401401
expect.objectContaining({
402402
Body: expect.stringContaining(JSON.stringify(params.expectedReplicationInfo)),
403403
}),
@@ -436,8 +436,8 @@ describe('ReplicationStatusUpdater with specifics', () => {
436436
constructor: expect.objectContaining({ name: 'GetBucketReplicationCommand' })
437437
}));
438438

439-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(1);
440-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
439+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(1);
440+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
441441

442442
assert.strictEqual(crr._nProcessed, 1);
443443
assert.strictEqual(crr._nSkipped, 0);
@@ -486,8 +486,8 @@ describe('ReplicationStatusUpdater with specifics', () => {
486486
constructor: expect.objectContaining({ name: 'GetBucketReplicationCommand' })
487487
}));
488488

489-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(2);
490-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(2);
489+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(2);
490+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(2);
491491

492492
assert.strictEqual(crr._nProcessed, 2);
493493
assert.strictEqual(crr._nSkipped, 0);
@@ -517,8 +517,8 @@ describe('ReplicationStatusUpdater with specifics', () => {
517517
expect(crr.s3.send).toHaveBeenNthCalledWith(2, expect.objectContaining({
518518
constructor: expect.objectContaining({ name: 'GetBucketReplicationCommand' })
519519
}));
520-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(1);
521-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
520+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(1);
521+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
522522

523523
assert.strictEqual(crr._nProcessed, 1);
524524
assert.strictEqual(crr._nSkipped, 0);
@@ -615,17 +615,17 @@ describe('ReplicationStatusUpdater with forceUsingConfiguration', () => {
615615
}, logger);
616616

617617
// Override getMetadata to return object with old replication info
618-
crr.bb.getMetadata = jest.fn((params, cb) => cb(null, {
618+
crr.cloudserverclient.getMetadata = jest.fn((params, cb) => cb(null, {
619619
Body: JSON.stringify(objectMdWithOldReplication),
620620
}));
621621

622622
crr.run(err => {
623623
assert.ifError(err);
624624

625-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
625+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
626626

627627
// Verify that putMetadata was called with updated destination and role from bucket config
628-
const putMetadataCall = crr.bb.putMetadata.mock.calls[0][0];
628+
const putMetadataCall = crr.cloudserverclient.putMetadata.mock.calls[0][0];
629629
const updatedMetadata = JSON.parse(putMetadataCall.Body);
630630

631631
// Check that replicationInfo contains the bucket configuration values
@@ -729,19 +729,19 @@ describe('ReplicationStatusUpdater with currentVersionOnly', () => {
729729
constructor: expect.objectContaining({ name: 'GetBucketReplicationCommand' })
730730
}));
731731

732-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(2);
733-
expect(crr.bb.getMetadata).toHaveBeenNthCalledWith(1, {
732+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(2);
733+
expect(crr.cloudserverclient.getMetadata).toHaveBeenNthCalledWith(1, {
734734
Bucket: 'bucket0',
735735
Key: 'key0',
736736
VersionId: 'aJdO148N3LjN00000000001I4j3QKItW',
737737
}, expect.any(Function));
738-
expect(crr.bb.getMetadata).toHaveBeenNthCalledWith(2, {
738+
expect(crr.cloudserverclient.getMetadata).toHaveBeenNthCalledWith(2, {
739739
Bucket: 'bucket0',
740740
Key: 'key1',
741741
VersionId: 'aJdO148N3LjN00000000001I4j3QKItU',
742742
}, expect.any(Function));
743743

744-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(2);
744+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(2);
745745

746746
assert.strictEqual(crr._nProcessed, 2);
747747
assert.strictEqual(crr._nSkipped, 2);
@@ -810,8 +810,8 @@ describe('ReplicationStatusUpdater with currentVersionOnly', () => {
810810
constructor: expect.objectContaining({ name: 'GetBucketReplicationCommand' })
811811
}));
812812

813-
expect(crr.bb.getMetadata).toHaveBeenCalledTimes(2);
814-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(2);
813+
expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(2);
814+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(2);
815815

816816
assert.strictEqual(crr._nProcessed, 2);
817817
assert.strictEqual(crr._nSkipped, 0);
@@ -847,7 +847,7 @@ describe('ReplicationStatusUpdater model version guard', () => {
847847
crr.run(err => {
848848
assert.ifError(err);
849849

850-
expect(crr.bb.putMetadata).not.toHaveBeenCalled();
850+
expect(crr.cloudserverclient.putMetadata).not.toHaveBeenCalled();
851851

852852
assert.strictEqual(crr._nProcessed, 1);
853853
assert.strictEqual(crr._nUpdated, 0);
@@ -871,7 +871,7 @@ describe('ReplicationStatusUpdater model version guard', () => {
871871
crr.run(err => {
872872
assert.ifError(err);
873873

874-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
874+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
875875

876876
assert.strictEqual(crr._nProcessed, 1);
877877
assert.strictEqual(crr._nUpdated, 1);
@@ -895,7 +895,7 @@ describe('ReplicationStatusUpdater model version guard', () => {
895895
crr.run(err => {
896896
assert.ifError(err);
897897

898-
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
898+
expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1);
899899

900900
assert.strictEqual(crr._nProcessed, 1);
901901
assert.strictEqual(crr._nUpdated, 1);

0 commit comments

Comments
 (0)