From 84b848df5502239bae7c902953dc6dd8955b3689 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 25 Mar 2025 09:55:29 +0100 Subject: [PATCH 1/4] Support per-location metrics in buckets Issue: S3UTILS-170 --- CountItems/CountManager.js | 25 +++++++----- utils/S3UtilsMongoClient.js | 80 +++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/CountItems/CountManager.js b/CountItems/CountManager.js index 625b9e39..59288cc3 100644 --- a/CountItems/CountManager.js +++ b/CountItems/CountManager.js @@ -9,7 +9,10 @@ class CountManager { this.log = params.log; this.workers = params.workers; this.maxConcurrent = params.maxConcurrent; - this.temporaryStore = {}; + this.temporaryStore = { + account: {}, + bucket: {}, + }; this.store = { objects: 0, versions: 0, @@ -94,14 +97,14 @@ class CountManager { this.dataMetrics[metricLevel][resourceName], results.dataMetrics[metricLevel][resourceName], ); - // if metricLevel is account, add the locations details - if (metricLevel === 'account') { + // if metricLevel is account or bucket, add the locations details + if (metricLevel === 'account' || metricLevel === 'bucket') { Object.keys((results.dataMetrics[metricLevel][resourceName].locations || {})).forEach(locationName => { - if (!this.temporaryStore[resourceName]) { - this.temporaryStore[resourceName] = {}; + if (!this.temporaryStore[metricLevel][resourceName]) { + this.temporaryStore[metricLevel][resourceName] = {}; } - this.temporaryStore[resourceName][locationName] = consolidateDataMetrics( - this.temporaryStore[resourceName][locationName], + this.temporaryStore[metricLevel][resourceName][locationName] = consolidateDataMetrics( + this.temporaryStore[metricLevel][resourceName][locationName], results.dataMetrics[metricLevel][resourceName].locations[locationName], ); }); @@ -110,8 +113,12 @@ class CountManager { } }); // Add the accounts details for locations from the temporary store - Object.keys(this.temporaryStore).forEach(accountName => { - this.dataMetrics.account[accountName].locations = this.temporaryStore[accountName]; + Object.keys(this.temporaryStore.account).forEach(accountName => { + this.dataMetrics.account[accountName].locations = this.temporaryStore.account[accountName]; + }); + // Add the locations details for buckets from the temporary store + Object.keys(this.temporaryStore.bucket).forEach(bucketName => { + this.dataMetrics.bucket[bucketName].locations = this.temporaryStore.bucket[bucketName]; }); } else { this.dataMetrics = results.dataMetrics; diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index f3374db5..b2061a85 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -324,6 +324,7 @@ class S3UtilsMongoClient extends MongoClientInterface { if (!collRes[metricLevel][resourceName]) { collRes[metricLevel][resourceName] = { ...baseMetricsObject, + ...(metricLevel === 'bucket' || metricLevel === 'account' ? { locations: {} } : {}), }; } collRes[metricLevel][resourceName][targetData] += BigInt(data[metricLevel][resourceName]); @@ -335,6 +336,24 @@ class S3UtilsMongoClient extends MongoClientInterface { }); } }); + // Handle bucket locations + Object.keys(data.bucket).forEach(bucket => { + if (!collRes.bucket[bucket].locations) { + collRes.bucket[bucket].locations = {}; + } + Object.keys(data.location).forEach(location => { + if (!collRes.bucket[bucket].locations[location]) { + collRes.bucket[bucket].locations[location] = { + ...baseMetricsObject, + }; + } + collRes.bucket[bucket].locations[location][targetData] += BigInt(data.location[location]); + if (!isMPUPart) { + collRes.bucket[bucket].locations[location][targetCount]++; + } + collRes.bucket[bucket].locations[location].deleteMarkerCount += entry.value.isDeleteMarker ? 1n : 0n; + }); + }); Object.keys(data.account).forEach(account => { if (!collRes.account[account].locations) { collRes.account[account].locations = {}; @@ -765,6 +784,67 @@ class S3UtilsMongoClient extends MongoClientInterface { }); }); + // parse all location and reflect the data in the bucket + Object.keys((res.bucket || {})).forEach(bucket => { + if (!dataMetrics.bucket[bucket].locations) { + dataMetrics.bucket[bucket].locations = {}; + } + Object.keys(res.location || {}).forEach(location => { + if (!dataMetrics.bucket[bucket].locations[location]) { + dataMetrics.bucket[bucket].locations[location] = {}; + } + const bucketLocation = dataMetrics.bucket[bucket].locations[location]; + if (!bucketLocation.usedCapacity) { + bucketLocation.usedCapacity = { + current: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }; + } + if (!bucketLocation.objectCount) { + bucketLocation.objectCount = { + current: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + deleteMarker: 0n, + }; + } + bucketLocation.usedCapacity.current += dataMetrics.location[location].usedCapacity.current; + bucketLocation.usedCapacity.nonCurrent += dataMetrics.location[location].usedCapacity.nonCurrent; + bucketLocation.usedCapacity._currentCold += dataMetrics.location[location].usedCapacity._currentCold; + bucketLocation.usedCapacity._nonCurrentCold += dataMetrics.location[location].usedCapacity._nonCurrentCold; + bucketLocation.usedCapacity._currentRestoring += dataMetrics.location[location].usedCapacity._currentRestoring; + bucketLocation.usedCapacity._nonCurrentRestoring += dataMetrics.location[location].usedCapacity._nonCurrentRestoring; + bucketLocation.usedCapacity._currentRestored += dataMetrics.location[location].usedCapacity._currentRestored; + bucketLocation.usedCapacity._nonCurrentRestored += dataMetrics.location[location].usedCapacity._nonCurrentRestored; + bucketLocation.usedCapacity._incompleteMPUParts += dataMetrics.location[location].usedCapacity._incompleteMPUParts; + + bucketLocation.objectCount.current += dataMetrics.location[location].objectCount.current; + bucketLocation.objectCount.nonCurrent += dataMetrics.location[location].objectCount.nonCurrent; + bucketLocation.objectCount._currentCold += dataMetrics.location[location].objectCount._currentCold; + bucketLocation.objectCount._nonCurrentCold += dataMetrics.location[location].objectCount._nonCurrentCold; + bucketLocation.objectCount._currentRestoring += dataMetrics.location[location].objectCount._currentRestoring; + bucketLocation.objectCount._nonCurrentRestoring += dataMetrics.location[location].objectCount._nonCurrentRestoring; + bucketLocation.objectCount._currentRestored += dataMetrics.location[location].objectCount._currentRestored; + bucketLocation.objectCount._nonCurrentRestored += dataMetrics.location[location].objectCount._nonCurrentRestored; + bucketLocation.objectCount._incompleteMPUUploads += dataMetrics.location[location].objectCount._incompleteMPUUploads; + + bucketLocation.objectCount.deleteMarker += dataMetrics.location[location].objectCount.deleteMarker; + }); + }); + return { // We do not support bigint for the global count items document versions: Number(totalNonCurrentCount + totalNonCurrentColdCount + totalVersionRestoringCount + totalVerionsRestoredCount), From cb70e9b73f69b92705b94e1b714f5a57bd083ff6 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 25 Mar 2025 09:55:56 +0100 Subject: [PATCH 2/4] Update tests Issue: S3UTILS-170 --- tests/functional/countItems.js | 14 + tests/functional/utils/S3UtilsMongoClient.js | 264 ++++++++++++ tests/unit/CountItems/CountManager.js | 167 ++++++++ tests/unit/utils/S3UtilsMongoClient.js | 416 +++++++++++++++++++ 4 files changed, 861 insertions(+) diff --git a/tests/functional/countItems.js b/tests/functional/countItems.js index 77fde394..a72079c5 100644 --- a/tests/functional/countItems.js +++ b/tests/functional/countItems.js @@ -53,6 +53,20 @@ const expectedDataMetrics = { [`bucket_test-bucket-0_${testBucketCreationDate}`]: { objectCount: { current: 10n, deleteMarker: 0n, nonCurrent: 0n }, usedCapacity: { current: 1000n, nonCurrent: 0n }, + locations: { + 'secondary-location-1': { + objectCount: { current: 30n, deleteMarker: 0n, nonCurrent: 30n }, + usedCapacity: { current: 3000n, nonCurrent: 3000n }, + }, + 'secondary-location-2': { + objectCount: { current: 30n, deleteMarker: 0n, nonCurrent: 30n }, + usedCapacity: { current: 3000n, nonCurrent: 3000n }, + }, + 'us-east-1': { + objectCount: { current: 90n, deleteMarker: 0n, nonCurrent: 60n }, + usedCapacity: { current: 9000n, nonCurrent: 6000n }, + }, + }, }, [`bucket_test-bucket-1_${testBucketCreationDate}`]: { objectCount: { current: 10n, deleteMarker: 0n, nonCurrent: 0n }, diff --git a/tests/functional/utils/S3UtilsMongoClient.js b/tests/functional/utils/S3UtilsMongoClient.js index 6cadb445..81db34f9 100644 --- a/tests/functional/utils/S3UtilsMongoClient.js +++ b/tests/functional/utils/S3UtilsMongoClient.js @@ -222,6 +222,33 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -398,6 +425,33 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 1n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -578,6 +632,33 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 0n, + deleteMarker: 1n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + nonCurrent: 20n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -772,6 +853,58 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 1n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 10n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'completed': { + objectCount: { + current: 1n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 10n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1005,6 +1138,58 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'completed': { + objectCount: { + current: 1n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 10n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1240,6 +1425,33 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 1n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1485,6 +1697,58 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => { }, bucket: { [`${BUCKET_NAME}_${BUCKET_CREATE_DATE}`]: { + locations: { + 'cold-location': { + objectCount: { + current: 0n, + _currentCold: 0n, + deleteMarker: 0n, + nonCurrent: 0n, + _nonCurrentCold: 1n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 1n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + _currentCold: 0n, + nonCurrent: 0n, + _nonCurrentCold: 10n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 10n, + _incompleteMPUParts: 0n, + }, + }, + 'us-east-1': { + objectCount: { + current: 0n, + _currentCold: 0n, + deleteMarker: 0n, + nonCurrent: 0n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 1n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + _currentCold: 0n, + nonCurrent: 0n, + _nonCurrentCold: 0n, + _currentRestored: 10n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 10n, + _incompleteMPUParts: 0n, + }, + }, + }, objectCount: { current: 0n, _currentCold: 0n, diff --git a/tests/unit/CountItems/CountManager.js b/tests/unit/CountItems/CountManager.js index 1a052685..e3c7d052 100644 --- a/tests/unit/CountItems/CountManager.js +++ b/tests/unit/CountItems/CountManager.js @@ -159,6 +159,33 @@ describe('CountItems::CountManager', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + location1: { + objectCount: { + current: 10n, + deleteMarker: 0n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 100n, + nonCurrent: 100n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 100n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -273,6 +300,34 @@ describe('CountItems::CountManager', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + location1: { + objectCount: { + current: 11n, + deleteMarker: 0n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 200n, + nonCurrent: 100n, + _inflightsPreScan: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 100n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -402,6 +457,34 @@ describe('CountItems::CountManager', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + location1: { + objectCount: { + current: 10n, + deleteMarker: 0n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 100n, + nonCurrent: 100n, + _inflightsPreScan: 1000n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 100n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -517,6 +600,34 @@ describe('CountItems::CountManager', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + location1: { + objectCount: { + current: 11n, + deleteMarker: 0n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 200n, + nonCurrent: 100n, + _inflightsPreScan: 1000n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 100n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -641,6 +752,34 @@ describe('CountItems::CountManager', () => { _incompleteMPUParts: 9007199254740991n, _inflightsPreScan: 9007199254740991n, }, + locations: { + location1: { + objectCount: { + current: 9007199254740991n, + deleteMarker: 9007199254740991n, + nonCurrent: 9007199254740991n, + _currentCold: 9007199254740991n, + _nonCurrentCold: 9007199254740991n, + _currentRestored: 9007199254740991n, + _currentRestoring: 9007199254740991n, + _nonCurrentRestored: 9007199254740991n, + _nonCurrentRestoring: 9007199254740991n, + _incompleteMPUUploads: 9007199254740991n, + }, + usedCapacity: { + current: 9007199254740991n, + nonCurrent: 9007199254740992n, + _currentCold: 9007199254740991n, + _nonCurrentCold: 9007199254740991n, + _currentRestored: 9007199254740991n, + _currentRestoring: 9007199254740991n, + _nonCurrentRestored: 9007199254740991n, + _nonCurrentRestoring: 9007199254740991n, + _incompleteMPUParts: 9007199254740991n, + _inflightsPreScan: 9007199254740991n, + }, + }, + }, }, }, location: { @@ -756,6 +895,34 @@ describe('CountItems::CountManager', () => { _incompleteMPUParts: 9007199254740991n, _inflightsPreScan: 9007199254740991n, }, + locations: { + location1: { + objectCount: { + current: 45035996273704955n, + deleteMarker: 9007199254740991n, + nonCurrent: 36028797018963964n, + _currentCold: 9007199254740991n, + _nonCurrentCold: 9007199254740991n, + _currentRestored: 9007199254740991n, + _currentRestoring: 9007199254740991n, + _nonCurrentRestored: 9007199254740991n, + _nonCurrentRestoring: 9007199254740991n, + _incompleteMPUUploads: 9007199254740991n, + }, + usedCapacity: { + current: 45035996273704955n, + nonCurrent: 36028797018963965n, + _currentCold: 9007199254740991n, + _nonCurrentCold: 9007199254740991n, + _currentRestored: 9007199254740991n, + _currentRestoring: 9007199254740991n, + _nonCurrentRestored: 9007199254740991n, + _nonCurrentRestoring: 9007199254740991n, + _incompleteMPUParts: 9007199254740991n, + _inflightsPreScan: 9007199254740991n, + }, + }, + }, }, }, location: { diff --git a/tests/unit/utils/S3UtilsMongoClient.js b/tests/unit/utils/S3UtilsMongoClient.js index b832d0cb..8b225c38 100644 --- a/tests/unit/utils/S3UtilsMongoClient.js +++ b/tests/unit/utils/S3UtilsMongoClient.js @@ -150,6 +150,58 @@ describe('S3UtilsMongoClient::_handleResults', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + location1: { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + location2: { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -326,6 +378,58 @@ describe('S3UtilsMongoClient::_handleResults', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + location1: { + objectCount: { + current: 2n, + deleteMarker: 1n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + location2: { + objectCount: { + current: 2n, + deleteMarker: 1n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 20n, + nonCurrent: 10n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1289,6 +1393,58 @@ describe('S3UtilsMongoClient, tests', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'rep-loc-1': { + objectCount: { + current: 0n, + deleteMarker: 0n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + nonCurrent: 200n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 200n, + nonCurrent: 200n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1465,6 +1621,33 @@ describe('S3UtilsMongoClient, tests', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 200n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1661,6 +1844,58 @@ describe('S3UtilsMongoClient, tests', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'rep-loc-1': { + objectCount: { + current: 0n, + deleteMarker: 1n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + nonCurrent: 300n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'us-east-1': { + objectCount: { + current: 1n, + deleteMarker: 1n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 100n, + nonCurrent: 300n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -1928,6 +2163,83 @@ describe('S3UtilsMongoClient, tests', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'rep-loc-1': { + objectCount: { + current: 0n, + _currentCold: 0n, + deleteMarker: 1n, + nonCurrent: 1n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + _currentCold: 0n, + nonCurrent: 100n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'us-east-1': { + objectCount: { + current: 1n, + _currentCold: 0n, + deleteMarker: 1n, + nonCurrent: 1n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 1n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 100n, + _currentCold: 0n, + nonCurrent: 100n, + _nonCurrentCold: 0n, + _currentRestored: 100n, + _currentRestoring: 100n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'cold-location': { + objectCount: { + current: 0n, + _currentCold: 1n, + deleteMarker: 0n, + nonCurrent: 0n, + _nonCurrentCold: 1n, + _currentRestored: 0n, + _currentRestoring: 1n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + _currentCold: 100n, + nonCurrent: 0n, + _nonCurrentCold: 100n, + _currentRestored: 0n, + _currentRestoring: 100n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -2153,6 +2465,58 @@ describe('S3UtilsMongoClient, tests', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'cold-location': { + objectCount: { + current: 0n, + _currentCold: 0n, + deleteMarker: 0n, + nonCurrent: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 1n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + _currentCold: 0n, + nonCurrent: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 100n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'us-east-1': { + objectCount: { + current: 0n, + _currentCold: 0n, + deleteMarker: 0n, + nonCurrent: 0n, + _nonCurrentCold: 0n, + _currentRestored: 1n, + _currentRestoring: 1n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + _currentCold: 0n, + nonCurrent: 0n, + _nonCurrentCold: 0n, + _currentRestored: 100n, + _currentRestoring: 100n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { @@ -2380,6 +2744,58 @@ describe('S3UtilsMongoClient, tests', () => { _nonCurrentRestoring: 0n, _incompleteMPUParts: 0n, }, + locations: { + 'rep-loc-1': { + objectCount: { + current: 0n, + deleteMarker: 0n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 0n, + nonCurrent: 200n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + 'us-east-1': { + objectCount: { + current: 2n, + deleteMarker: 0n, + nonCurrent: 2n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + }, + usedCapacity: { + current: 200n, + nonCurrent: 200n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }, + }, + }, }, }, location: { From 0fdced9e87a1bfc71093692a1fa35b37b34d4e32 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 26 Mar 2025 10:12:19 +0100 Subject: [PATCH 3/4] Add a comment to explain why we have duplicated code Issue: S3UTILS-170 --- utils/S3UtilsMongoClient.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index b2061a85..6fd4ebe1 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -734,6 +734,8 @@ class S3UtilsMongoClient extends MongoClientInterface { } const accountLocation = dataMetrics.account[account].locations[location]; if (!accountLocation.usedCapacity) { + // the capacities are created inline to avoid runtime + // deep copy of the baseMetricsObject accountLocation.usedCapacity = { current: 0n, nonCurrent: 0n, From 6c7851fbbf11b51ad35a609bb6ce7d3b7b09bdc4 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 25 Mar 2025 09:56:14 +0100 Subject: [PATCH 4/4] Bump S3Utils Issue: S3UTILS-170 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a29ffe1f..ca32a78a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.15.7", + "version": "1.15.8", "engines": { "node": ">= 16" },