Skip to content

Commit 0b3e1b1

Browse files
bert-ebert-e
committed
Merge branch 'improvement/S3UTILS-208/FORCE_USING_CONFIGURATION' into q/1.16
2 parents 67449fb + 2f27cab commit 0b3e1b1

File tree

7 files changed

+363
-45
lines changed

7 files changed

+363
-45
lines changed

‎CRR/ReplicationStatusUpdater.js‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class ReplicationStatusUpdater {
3030
* @param {string} [params.keyMarker] - (Optional) Key marker for resuming object listing.
3131
* @param {string} [params.versionIdMarker] - (Optional) Version ID marker for resuming object listing.
3232
* @param {boolean} [params.currentVersionOnly] - (Optional) Whether to process only the current version of objects.
33+
* @param {boolean} [params.forceUsingConfiguration] - (Optional) Force reset replication target to bucket's configuration.
3334
*/
3435
constructor(params, log) {
3536
const {
@@ -48,6 +49,7 @@ class ReplicationStatusUpdater {
4849
keyMarker,
4950
versionIdMarker,
5051
currentVersionOnly,
52+
forceUsingConfiguration,
5153
} = params;
5254

5355
// inputs
@@ -66,6 +68,7 @@ class ReplicationStatusUpdater {
6668
this.inputKeyMarker = keyMarker;
6769
this.inputVersionIdMarker = versionIdMarker;
6870
this.currentVersionOnly = currentVersionOnly;
71+
this.forceUsingConfiguration = forceUsingConfiguration;
6972
this.log = log;
7073

7174
this._setupClients();
@@ -188,9 +191,10 @@ class ReplicationStatusUpdater {
188191
// This is particularly important if the object was created before
189192
// enabling replication on the bucket.
190193
let replicationInfo = objMD.getReplicationInfo();
194+
const { Rules, Role } = repConfig;
195+
const destination = Rules[0].Destination.Bucket;
196+
191197
if (!replicationInfo || !replicationInfo.status) {
192-
const { Rules, Role } = repConfig;
193-
const destination = Rules[0].Destination.Bucket;
194198
// set replication properties
195199
const ops = objMD.getContentLength() === 0 ? ['METADATA']
196200
: ['METADATA', 'DATA'];
@@ -205,6 +209,12 @@ class ReplicationStatusUpdater {
205209
};
206210
objMD.setReplicationInfo(replicationInfo);
207211
}
212+
213+
// Force reset object's replication configuration to match bucket's configuration
214+
if (this.forceUsingConfiguration) {
215+
objMD.setReplicationTargetBucket(destination);
216+
objMD.setReplicationRoles(Role);
217+
}
208218
// Update replication info with site specific info
209219
if (!objMD.getReplicationSiteStatus(storageClass)) {
210220
// When replicating to multiple destinations,

‎crrExistingObjects.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const { SITE_NAME } = process.env;
1515
let { STORAGE_TYPE } = process.env;
1616
let { TARGET_REPLICATION_STATUS } = process.env;
1717
const { TARGET_PREFIX } = process.env;
18+
const FORCE_USING_CONFIGURATION = process.env.FORCE_USING_CONFIGURATION === 'true' || process.env.FORCE_USING_CONFIGURATION === '1' || false;
1819
const WORKERS = (process.env.WORKERS
1920
&& Number.parseInt(process.env.WORKERS, 10)) || 10;
2021
const MAX_UPDATES = (process.env.MAX_UPDATES
@@ -92,6 +93,7 @@ const replicationStatusUpdater = new ReplicationStatusUpdater({
9293
keyMarker: KEY_MARKER,
9394
versionIdMarker: VERSION_ID_MARKER,
9495
currentVersionOnly: CURRENT_VERSION_ONLY,
96+
forceUsingConfiguration: FORCE_USING_CONFIGURATION,
9597
}, log);
9698

9799
replicationStatusUpdater.run(err => {

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "s3utils",
3-
"version": "1.16.1",
3+
"version": "1.16.2",
44
"engines": {
55
"node": ">= 22"
66
},
@@ -31,7 +31,7 @@
3131
"@senx/warp10": "^2.0.3",
3232
"@smithy/util-retry": "^4.0.7",
3333
"JSONStream": "^1.3.5",
34-
"arsenal": "git+https://github.com/scality/arsenal#8.2.26",
34+
"arsenal": "git+https://github.com/scality/arsenal#8.2.36",
3535
"async": "^3.2.6",
3636
"aws-sdk": "^2.1691.0",
3737
"bucketclient": "scality/bucketclient#8.2.6",

‎tests/unit/CRR/ReplicationStatusUpdater.js‎

Lines changed: 80 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,25 +7,22 @@ const {
77
listVersionsRes,
88
listVersionWithMarkerRes,
99
getMetadataRes,
10+
objectMd,
1011
} = require('../../utils/crr');
1112

1213
const logger = new werelogs.Logger('ReplicationStatusUpdater::tests', 'debug', 'debug');
1314

1415
describe('ReplicationStatusUpdater', () => {
15-
let crr;
16-
17-
beforeEach(() => {
18-
crr = initializeCrrWithMocks({
16+
it('should process bucket for CRR', done => {
17+
const crr = initializeCrrWithMocks({
1918
buckets: ['bucket0'],
2019
workers: 10,
2120
replicationStatusToProcess: ['NEW'],
2221
targetPrefix: 'toto',
2322
listingLimit: 10,
2423
siteName: 'aws-location',
2524
}, logger);
26-
});
2725

28-
it('should process bucket for CRR', done => {
2926
crr.run(err => {
3027
assert.ifError(err);
3128

@@ -66,7 +63,7 @@ describe('ReplicationStatusUpdater', () => {
6663
},
6764
],
6865
content: ['METADATA', 'DATA'],
69-
destination: 'arn:aws:s3:::sourcebucket',
66+
destination: 'arn:aws:s3:::destination',
7067
storageClass: 'aws-location',
7168
role: 'arn:aws:iam::root:role/s3-replication-role',
7269
storageType: '',
@@ -88,7 +85,7 @@ describe('ReplicationStatusUpdater', () => {
8885
});
8986

9087
it('should process bucket for CRR with multiple objects', done => {
91-
crr = initializeCrrWithMocks({
88+
const crr = initializeCrrWithMocks({
9289
buckets: ['bucket0'],
9390
workers: 10,
9491
replicationStatusToProcess: ['NEW'],
@@ -159,7 +156,7 @@ describe('ReplicationStatusUpdater', () => {
159156
},
160157
],
161158
content: ['METADATA', 'DATA'],
162-
destination: 'arn:aws:s3:::sourcebucket',
159+
destination: 'arn:aws:s3:::destination',
163160
storageClass: 'aws-location',
164161
role: 'arn:aws:iam::root:role/s3-replication-role',
165162
storageType: 'aws_s3',
@@ -188,7 +185,7 @@ describe('ReplicationStatusUpdater', () => {
188185
},
189186
],
190187
content: ['METADATA', 'DATA'],
191-
destination: 'arn:aws:s3:::sourcebucket',
188+
destination: 'arn:aws:s3:::destination',
192189
storageClass: 'aws-location',
193190
role: 'arn:aws:iam::root:role/s3-replication-role',
194191
storageType: 'aws_s3',
@@ -206,7 +203,7 @@ describe('ReplicationStatusUpdater', () => {
206203
},
207204
],
208205
content: ['METADATA', 'DATA'],
209-
destination: 'arn:aws:s3:::sourcebucket',
206+
destination: 'arn:aws:s3:::destination',
210207
storageClass: 'aws-location',
211208
role: 'arn:aws:iam::root:role/s3-replication-role',
212209
storageType: 'aws_s3',
@@ -223,7 +220,7 @@ describe('ReplicationStatusUpdater', () => {
223220
},
224221
],
225222
content: ['METADATA', 'DATA'],
226-
destination: 'arn:aws:s3:::sourcebucket',
223+
destination: 'arn:aws:s3:::destination',
227224
storageClass: 'aws-location',
228225
role: 'arn:aws:iam::root:role/s3-replication-role',
229226
storageType: 'aws_s3',
@@ -241,7 +238,7 @@ describe('ReplicationStatusUpdater', () => {
241238
},
242239
],
243240
content: ['METADATA', 'DATA'],
244-
destination: 'arn:aws:s3:::sourcebucket',
241+
destination: 'arn:aws:s3:::destination',
245242
storageClass: 'aws-location',
246243
role: 'arn:aws:iam::root:role/s3-replication-role',
247244
storageType: 'aws_s3',
@@ -258,7 +255,7 @@ describe('ReplicationStatusUpdater', () => {
258255
},
259256
],
260257
content: ['METADATA', 'DATA'],
261-
destination: 'arn:aws:s3:::sourcebucket',
258+
destination: 'arn:aws:s3:::destination',
262259
storageClass: 'aws-location',
263260
role: 'arn:aws:iam::root:role/s3-replication-role',
264261
storageType: 'aws_s3',
@@ -281,7 +278,7 @@ describe('ReplicationStatusUpdater', () => {
281278
},
282279
],
283280
content: ['METADATA', 'DATA'],
284-
destination: 'arn:aws:s3:::sourcebucket',
281+
destination: 'arn:aws:s3:::destination',
285282
storageClass: 'azure-location,aws-location',
286283
role: 'arn:aws:iam::root:role/s3-replication-role',
287284
storageType: 'azure,aws_s3',
@@ -302,7 +299,7 @@ describe('ReplicationStatusUpdater', () => {
302299
},
303300
],
304301
content: ['METADATA', 'DATA'],
305-
destination: 'arn:aws:s3:::sourcebucket',
302+
destination: 'arn:aws:s3:::destination',
306303
storageClass: 'azure-location,aws-location',
307304
role: 'arn:aws:iam::root:role/s3-replication-role',
308305
storageType: 'azure,aws_s3',
@@ -325,7 +322,7 @@ describe('ReplicationStatusUpdater', () => {
325322
},
326323
],
327324
content: ['METADATA', 'DATA'],
328-
destination: 'arn:aws:s3:::sourcebucket',
325+
destination: 'arn:aws:s3:::destination',
329326
storageClass: 'azure-location,azure-location-2',
330327
role: 'arn:aws:iam::root:role/s3-replication-role',
331328
storageType: 'azure,azure',
@@ -350,7 +347,7 @@ describe('ReplicationStatusUpdater', () => {
350347
},
351348
],
352349
content: ['METADATA', 'DATA'],
353-
destination: 'arn:aws:s3:::sourcebucket',
350+
destination: 'arn:aws:s3:::destination',
354351
storageClass: 'azure-location,azure-location-2,aws-location',
355352
role: 'arn:aws:iam::root:role/s3-replication-role',
356353
storageType: 'azure,azure,aws_s3',
@@ -359,6 +356,14 @@ describe('ReplicationStatusUpdater', () => {
359356
},
360357
].forEach(params => {
361358
it(`should trigger replication ${params.description}`, done => {
359+
const crr = initializeCrrWithMocks({
360+
buckets: ['bucket0'],
361+
workers: 10,
362+
replicationStatusToProcess: ['NEW'],
363+
targetPrefix: 'toto',
364+
listingLimit: 10,
365+
siteName: 'aws-location',
366+
}, logger);
362367
crr.bb.getMetadata = jest.fn((p, cb) => {
363368
const objectMd = JSON.parse(getMetadataRes.Body);
364369
objectMd.replicationInfo = params.replicationInfo;
@@ -578,6 +583,63 @@ describe('ReplicationStatusUpdater with specifics', () => {
578583
});
579584
});
580585

586+
describe('ReplicationStatusUpdater with forceUsingConfiguration', () => {
587+
it('should overwrite replication destination and role when forceUsingConfiguration is true', done => {
588+
// Deep copy objectMd from utils to avoid affecting the original
589+
const objectMdWithOldReplication = JSON.parse(JSON.stringify(objectMd));
590+
591+
// Only modify the replicationInfo part
592+
objectMdWithOldReplication.replicationInfo = {
593+
status: 'COMPLETED',
594+
backends: [
595+
{
596+
site: 'sf',
597+
status: 'COMPLETED',
598+
dataStoreVersionId: '',
599+
},
600+
],
601+
content: ['DATA', 'METADATA'],
602+
destination: 'arn:aws:s3:::destination2',
603+
storageClass: 'sf',
604+
role: 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource',
605+
storageType: '',
606+
dataStoreVersionId: '',
607+
};
608+
609+
const crr = initializeCrrWithMocks({
610+
buckets: ['bucket0'],
611+
workers: 10,
612+
replicationStatusToProcess: ['NEW'],
613+
forceUsingConfiguration: true,
614+
}, logger);
615+
616+
// Override getMetadata to return object with old replication info
617+
crr.bb.getMetadata = jest.fn((params, cb) => cb(null, {
618+
Body: JSON.stringify(objectMdWithOldReplication),
619+
}));
620+
621+
crr.run(err => {
622+
assert.ifError(err);
623+
624+
expect(crr.bb.putMetadata).toHaveBeenCalledTimes(1);
625+
626+
// Verify that putMetadata was called with updated destination and role from bucket config
627+
const putMetadataCall = crr.bb.putMetadata.mock.calls[0][0];
628+
const updatedMetadata = JSON.parse(putMetadataCall.Body);
629+
630+
// Check that replicationInfo contains the bucket configuration values
631+
expect(updatedMetadata.replicationInfo.destination).toBe('arn:aws:s3:::destination');
632+
expect(updatedMetadata.replicationInfo.role).toBe('arn:aws:iam::root:role/s3-replication-role');
633+
634+
assert.strictEqual(crr._nProcessed, 1);
635+
assert.strictEqual(crr._nSkipped, 0);
636+
assert.strictEqual(crr._nUpdated, 1);
637+
assert.strictEqual(crr._nErrors, 0);
638+
done();
639+
});
640+
});
641+
});
642+
581643
describe('ReplicationStatusUpdater with currentVersionOnly', () => {
582644
it('should process only latest versions when currentVersionOnly is true', done => {
583645
const listVersionsWithMixedLatest = {

‎tests/unit/CRR/crrExistingObjects.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ describe('crrExistingObjects', () => {
7676
process.env.VERSION_ID_MARKER = 'testVersionIdMarker';
7777
process.env.DEBUG = '0';
7878
process.env.CURRENT_VERSION_ONLY = 'true';
79+
process.env.FORCE_USING_CONFIGURATION = 'true';
7980

8081
require('../../../crrExistingObjects');
8182

@@ -98,6 +99,7 @@ describe('crrExistingObjects', () => {
9899
keyMarker: 'testKeyMarker',
99100
versionIdMarker: 'testVersionIdMarker',
100101
currentVersionOnly: true,
102+
forceUsingConfiguration: true,
101103
}, expect.anything());
102104

103105
expect(mockFatal).not.toHaveBeenCalled();
@@ -134,6 +136,7 @@ describe('crrExistingObjects', () => {
134136
keyMarker: undefined,
135137
versionIdMarker: undefined,
136138
currentVersionOnly: false,
139+
forceUsingConfiguration: false,
137140
}, expect.anything());
138141

139142
expect(mockFatal).not.toHaveBeenCalled();

‎tests/utils/crr.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ const getBucketReplicationRes = {
9797
Prefix: '',
9898
Status: 'Enabled',
9999
Destination: {
100-
Bucket: 'arn:aws:s3:::sourcebucket',
100+
Bucket: 'arn:aws:s3:::destination',
101101
StorageClass: 'aws-location',
102102
},
103103
},
@@ -204,4 +204,5 @@ module.exports = {
204204
getBucketReplicationRes,
205205
getMetadataRes,
206206
putMetadataRes,
207+
objectMd,
207208
};

0 commit comments

Comments
 (0)