Skip to content

Commit 84b848d

Browse files
SeragoniaSeragonia
committed
Support per-location metrics in buckets
Issue: S3UTILS-170
1 parent 2086ecf commit 84b848d

File tree

2 files changed

+96
-9
lines changed

2 files changed

+96
-9
lines changed

‎CountItems/CountManager.js‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@ class CountManager {
99
this.log = params.log;
1010
this.workers = params.workers;
1111
this.maxConcurrent = params.maxConcurrent;
12-
this.temporaryStore = {};
12+
this.temporaryStore = {
13+
account: {},
14+
bucket: {},
15+
};
1316
this.store = {
1417
objects: 0,
1518
versions: 0,
@@ -94,14 +97,14 @@ class CountManager {
9497
this.dataMetrics[metricLevel][resourceName],
9598
results.dataMetrics[metricLevel][resourceName],
9699
);
97-
// if metricLevel is account, add the locations details
98-
if (metricLevel === 'account') {
100+
// if metricLevel is account or bucket, add the locations details
101+
if (metricLevel === 'account' || metricLevel === 'bucket') {
99102
Object.keys((results.dataMetrics[metricLevel][resourceName].locations || {})).forEach(locationName => {
100-
if (!this.temporaryStore[resourceName]) {
101-
this.temporaryStore[resourceName] = {};
103+
if (!this.temporaryStore[metricLevel][resourceName]) {
104+
this.temporaryStore[metricLevel][resourceName] = {};
102105
}
103-
this.temporaryStore[resourceName][locationName] = consolidateDataMetrics(
104-
this.temporaryStore[resourceName][locationName],
106+
this.temporaryStore[metricLevel][resourceName][locationName] = consolidateDataMetrics(
107+
this.temporaryStore[metricLevel][resourceName][locationName],
105108
results.dataMetrics[metricLevel][resourceName].locations[locationName],
106109
);
107110
});
@@ -110,8 +113,12 @@ class CountManager {
110113
}
111114
});
112115
// Add the accounts details for locations from the temporary store
113-
Object.keys(this.temporaryStore).forEach(accountName => {
114-
this.dataMetrics.account[accountName].locations = this.temporaryStore[accountName];
116+
Object.keys(this.temporaryStore.account).forEach(accountName => {
117+
this.dataMetrics.account[accountName].locations = this.temporaryStore.account[accountName];
118+
});
119+
// Add the locations details for buckets from the temporary store
120+
Object.keys(this.temporaryStore.bucket).forEach(bucketName => {
121+
this.dataMetrics.bucket[bucketName].locations = this.temporaryStore.bucket[bucketName];
115122
});
116123
} else {
117124
this.dataMetrics = results.dataMetrics;

‎utils/S3UtilsMongoClient.js‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
324324
if (!collRes[metricLevel][resourceName]) {
325325
collRes[metricLevel][resourceName] = {
326326
...baseMetricsObject,
327+
...(metricLevel === 'bucket' || metricLevel === 'account' ? { locations: {} } : {}),
327328
};
328329
}
329330
collRes[metricLevel][resourceName][targetData] += BigInt(data[metricLevel][resourceName]);
@@ -335,6 +336,24 @@ class S3UtilsMongoClient extends MongoClientInterface {
335336
});
336337
}
337338
});
339+
// Handle bucket locations
340+
Object.keys(data.bucket).forEach(bucket => {
341+
if (!collRes.bucket[bucket].locations) {
342+
collRes.bucket[bucket].locations = {};
343+
}
344+
Object.keys(data.location).forEach(location => {
345+
if (!collRes.bucket[bucket].locations[location]) {
346+
collRes.bucket[bucket].locations[location] = {
347+
...baseMetricsObject,
348+
};
349+
}
350+
collRes.bucket[bucket].locations[location][targetData] += BigInt(data.location[location]);
351+
if (!isMPUPart) {
352+
collRes.bucket[bucket].locations[location][targetCount]++;
353+
}
354+
collRes.bucket[bucket].locations[location].deleteMarkerCount += entry.value.isDeleteMarker ? 1n : 0n;
355+
});
356+
});
338357
Object.keys(data.account).forEach(account => {
339358
if (!collRes.account[account].locations) {
340359
collRes.account[account].locations = {};
@@ -765,6 +784,67 @@ class S3UtilsMongoClient extends MongoClientInterface {
765784
});
766785
});
767786

787+
// parse all location and reflect the data in the bucket
788+
Object.keys((res.bucket || {})).forEach(bucket => {
789+
if (!dataMetrics.bucket[bucket].locations) {
790+
dataMetrics.bucket[bucket].locations = {};
791+
}
792+
Object.keys(res.location || {}).forEach(location => {
793+
if (!dataMetrics.bucket[bucket].locations[location]) {
794+
dataMetrics.bucket[bucket].locations[location] = {};
795+
}
796+
const bucketLocation = dataMetrics.bucket[bucket].locations[location];
797+
if (!bucketLocation.usedCapacity) {
798+
bucketLocation.usedCapacity = {
799+
current: 0n,
800+
nonCurrent: 0n,
801+
_currentCold: 0n,
802+
_nonCurrentCold: 0n,
803+
_currentRestored: 0n,
804+
_currentRestoring: 0n,
805+
_nonCurrentRestored: 0n,
806+
_nonCurrentRestoring: 0n,
807+
_incompleteMPUParts: 0n,
808+
};
809+
}
810+
if (!bucketLocation.objectCount) {
811+
bucketLocation.objectCount = {
812+
current: 0n,
813+
nonCurrent: 0n,
814+
_currentCold: 0n,
815+
_nonCurrentCold: 0n,
816+
_currentRestored: 0n,
817+
_currentRestoring: 0n,
818+
_nonCurrentRestored: 0n,
819+
_nonCurrentRestoring: 0n,
820+
_incompleteMPUUploads: 0n,
821+
deleteMarker: 0n,
822+
};
823+
}
824+
bucketLocation.usedCapacity.current += dataMetrics.location[location].usedCapacity.current;
825+
bucketLocation.usedCapacity.nonCurrent += dataMetrics.location[location].usedCapacity.nonCurrent;
826+
bucketLocation.usedCapacity._currentCold += dataMetrics.location[location].usedCapacity._currentCold;
827+
bucketLocation.usedCapacity._nonCurrentCold += dataMetrics.location[location].usedCapacity._nonCurrentCold;
828+
bucketLocation.usedCapacity._currentRestoring += dataMetrics.location[location].usedCapacity._currentRestoring;
829+
bucketLocation.usedCapacity._nonCurrentRestoring += dataMetrics.location[location].usedCapacity._nonCurrentRestoring;
830+
bucketLocation.usedCapacity._currentRestored += dataMetrics.location[location].usedCapacity._currentRestored;
831+
bucketLocation.usedCapacity._nonCurrentRestored += dataMetrics.location[location].usedCapacity._nonCurrentRestored;
832+
bucketLocation.usedCapacity._incompleteMPUParts += dataMetrics.location[location].usedCapacity._incompleteMPUParts;
833+
834+
bucketLocation.objectCount.current += dataMetrics.location[location].objectCount.current;
835+
bucketLocation.objectCount.nonCurrent += dataMetrics.location[location].objectCount.nonCurrent;
836+
bucketLocation.objectCount._currentCold += dataMetrics.location[location].objectCount._currentCold;
837+
bucketLocation.objectCount._nonCurrentCold += dataMetrics.location[location].objectCount._nonCurrentCold;
838+
bucketLocation.objectCount._currentRestoring += dataMetrics.location[location].objectCount._currentRestoring;
839+
bucketLocation.objectCount._nonCurrentRestoring += dataMetrics.location[location].objectCount._nonCurrentRestoring;
840+
bucketLocation.objectCount._currentRestored += dataMetrics.location[location].objectCount._currentRestored;
841+
bucketLocation.objectCount._nonCurrentRestored += dataMetrics.location[location].objectCount._nonCurrentRestored;
842+
bucketLocation.objectCount._incompleteMPUUploads += dataMetrics.location[location].objectCount._incompleteMPUUploads;
843+
844+
bucketLocation.objectCount.deleteMarker += dataMetrics.location[location].objectCount.deleteMarker;
845+
});
846+
});
847+
768848
return {
769849
// We do not support bigint for the global count items document
770850
versions: Number(totalNonCurrentCount + totalNonCurrentColdCount + totalVersionRestoringCount + totalVerionsRestoredCount),

0 commit comments

Comments
 (0)