Skip to content

Commit 2086ecf

Browse files
SeragoniaSeragonia
committed
Merge remote-tracking branch 'origin/bugfix/S3UTILS-188-support-metrics-above-max-safe' into w/1.15/bugfix/S3UTILS-188-support-metrics-above-max-safe
2 parents e82f3ea + e79a2f6 commit 2086ecf

21 files changed

+4474
-5386
lines changed

‎.eslintrc‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"jest/globals": true
88
},
99
"parserOptions": {
10-
"ecmaVersion": 9
10+
"ecmaVersion": 2020
1111
},
1212
"rules": {
1313
"no-plusplus": 0,
@@ -23,7 +23,11 @@
2323
"no-lonely-if": 0,
2424
"max-classes-per-file": 0,
2525
"prefer-spread": 0,
26-
"no-constructor-return": 0
26+
"no-constructor-return": 0,
27+
"new-cap": 0
28+
},
29+
"globals": {
30+
"BigInt": "readonly"
2731
}
2832
}
2933

‎.github/workflows/build.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ jobs:
3939
username: ${{ github.repository_owner }}
4040
password: ${{ github.token }}
4141

42+
- name: Install Oras
43+
run: |
44+
curl -L https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz | \
45+
tar -xz -C /usr/local/bin oras
46+
env:
47+
ORAS_VERSION: 1.2.2
48+
4249
- name: Push dashboards into the development namespace
4350
run: |
4451
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ inputs.tag }} \

‎.github/workflows/test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ env:
1515

1616
jobs:
1717
prepare:
18-
runs-on: ubuntu-latest
18+
runs-on: ubuntu-22.04
1919
permissions:
2020
# Need to explicitely add package write permissions for dependabot
2121
contents: read
@@ -45,7 +45,7 @@ jobs:
4545

4646
tests:
4747
needs: prepare
48-
runs-on: ubuntu-latest
48+
runs-on: ubuntu-22.04
4949
services:
5050
mongodb:
5151
image: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}

‎CountItems/CountManager.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ class CountManager {
6161
if (!results) {
6262
return;
6363
}
64-
this.store.versions += results.versions;
65-
this.store.objects += results.objects;
64+
this.store.versions += (results.versions || 0);
65+
this.store.objects += (results.objects || 0);
6666
this.store.stalled += results.stalled;
6767
if (results.dataManaged
6868
&& results.dataManaged.locations

‎CountItems/CountWorker.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ const assert = require('assert');
22
const async = require('async');
33
const { BucketInfo } = require('arsenal').models;
44
const monitoring = require('../utils/monitoring');
5+
const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
56

67
class CountWorker {
78
constructor(params) {
@@ -61,7 +62,7 @@ class CountWorker {
6162
}
6263
switch (data.type) {
6364
case 'count':
64-
this.countItems(data.bucketInfo, (err, results) => {
65+
this.countItems(deserializeBigInts(data.bucketInfo), (err, results) => {
6566
if (err) {
6667
return this._sendFn({
6768
id: data.id,
@@ -76,7 +77,7 @@ class CountWorker {
7677
owner: 'scality',
7778
type: 'count',
7879
status: 'passed',
79-
results,
80+
results: serializeBigInts(results),
8081
});
8182
});
8283
break;

‎CountItems/CountWorkerObj.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const uuid = require('node-uuid');
22
const { once } = require('arsenal').jsutil;
3+
const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
34

45
class CountWorkerObj {
56
constructor(id, worker) {
@@ -118,12 +119,18 @@ class CountWorkerObj {
118119

119120
count(bucketInfo, callback) {
120121
const id = uuid.v4();
121-
this._addCallback(id, 'count', callback);
122+
this._addCallback(id, 'count', (err, results) => {
123+
if (err) {
124+
return callback(err);
125+
}
126+
// Deserialize BigInts from the worker response
127+
return callback(null, deserializeBigInts(results));
128+
});
122129
this._worker.send({
123130
id,
124131
owner: 'scality',
125132
type: 'count',
126-
bucketInfo,
133+
bucketInfo: serializeBigInts(bucketInfo),
127134
});
128135
}
129136

‎CountItems/utils/utils.js‎

Lines changed: 77 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -9,79 +9,110 @@ function consolidateDataMetrics(target, source) {
99
if (!resTarget.usedCapacity) {
1010
Object.assign(resTarget, {
1111
usedCapacity: {
12-
current: 0,
13-
nonCurrent: 0,
14-
_currentCold: 0,
15-
_nonCurrentCold: 0,
16-
_currentRestored: 0,
17-
_currentRestoring: 0,
18-
_nonCurrentRestored: 0,
19-
_nonCurrentRestoring: 0,
20-
_inflightsPreScan: 0,
21-
_incompleteMPUParts: 0,
12+
current: 0n,
13+
nonCurrent: 0n,
14+
_currentCold: 0n,
15+
_nonCurrentCold: 0n,
16+
_currentRestored: 0n,
17+
_currentRestoring: 0n,
18+
_nonCurrentRestored: 0n,
19+
_nonCurrentRestoring: 0n,
20+
_inflightsPreScan: 0n,
21+
_incompleteMPUParts: 0n,
2222
},
2323
});
2424
}
2525
if (!resTarget.objectCount) {
2626
Object.assign(resTarget, {
2727
objectCount: {
28-
current: 0,
29-
nonCurrent: 0,
30-
_currentCold: 0,
31-
_nonCurrentCold: 0,
32-
_currentRestored: 0,
33-
_currentRestoring: 0,
34-
_nonCurrentRestored: 0,
35-
_nonCurrentRestoring: 0,
36-
_incompleteMPUUploads: 0,
37-
deleteMarker: 0,
28+
current: 0n,
29+
nonCurrent: 0n,
30+
_currentCold: 0n,
31+
_nonCurrentCold: 0n,
32+
_currentRestored: 0n,
33+
_currentRestoring: 0n,
34+
_nonCurrentRestored: 0n,
35+
_nonCurrentRestoring: 0n,
36+
_incompleteMPUUploads: 0n,
37+
deleteMarker: 0n,
3838
},
3939
});
4040
}
4141
if (!source) {
4242
return resTarget;
4343
}
4444
const { usedCapacity, objectCount, accountOwnerID } = source;
45-
resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : 0;
46-
resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : 0;
47-
resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : 0;
48-
resTarget.usedCapacity._nonCurrentCold += usedCapacity && usedCapacity._nonCurrentCold ? usedCapacity._nonCurrentCold : 0;
49-
resTarget.usedCapacity._currentRestoring += usedCapacity && usedCapacity._currentRestoring ? usedCapacity._currentRestoring : 0;
50-
resTarget.usedCapacity._currentRestored += usedCapacity && usedCapacity._currentRestored ? usedCapacity._currentRestored : 0;
51-
resTarget.usedCapacity._nonCurrentRestoring += usedCapacity && usedCapacity._nonCurrentRestoring ? usedCapacity._nonCurrentRestoring : 0;
52-
resTarget.usedCapacity._nonCurrentRestored += usedCapacity && usedCapacity._nonCurrentRestored ? usedCapacity._nonCurrentRestored : 0;
53-
resTarget.usedCapacity._incompleteMPUParts += usedCapacity && usedCapacity._incompleteMPUParts ? usedCapacity._incompleteMPUParts : 0;
45+
resTarget.usedCapacity.current += BigInt(usedCapacity?.current || 0n);
46+
resTarget.usedCapacity.nonCurrent += BigInt(usedCapacity?.nonCurrent || 0n);
47+
resTarget.usedCapacity._currentCold += BigInt(usedCapacity?._currentCold || 0n);
48+
resTarget.usedCapacity._nonCurrentCold += BigInt(usedCapacity?._nonCurrentCold || 0n);
49+
resTarget.usedCapacity._currentRestoring += BigInt(usedCapacity?._currentRestoring || 0n);
50+
resTarget.usedCapacity._currentRestored += BigInt(usedCapacity?._currentRestored || 0n);
51+
resTarget.usedCapacity._nonCurrentRestoring += BigInt(usedCapacity?._nonCurrentRestoring || 0n);
52+
resTarget.usedCapacity._nonCurrentRestored += BigInt(usedCapacity?._nonCurrentRestored || 0n);
53+
resTarget.usedCapacity._incompleteMPUParts += BigInt(usedCapacity?._incompleteMPUParts || 0n);
5454

55-
resTarget.objectCount.current += objectCount && objectCount.current ? objectCount.current : 0;
56-
resTarget.objectCount.nonCurrent += objectCount && objectCount.nonCurrent ? objectCount.nonCurrent : 0;
57-
resTarget.objectCount.deleteMarker += objectCount && objectCount.deleteMarker ? objectCount.deleteMarker : 0;
58-
resTarget.objectCount._currentCold += objectCount && objectCount._currentCold ? objectCount._currentCold : 0;
59-
resTarget.objectCount._nonCurrentCold += objectCount && objectCount._nonCurrentCold ? objectCount._nonCurrentCold : 0;
60-
resTarget.objectCount._currentRestoring += objectCount && objectCount._currentRestoring ? objectCount._currentRestoring : 0;
61-
resTarget.objectCount._currentRestored += objectCount && objectCount._currentRestored ? objectCount._currentRestored : 0;
62-
resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : 0;
63-
resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : 0;
64-
resTarget.objectCount._incompleteMPUUploads += objectCount && objectCount._incompleteMPUUploads ? objectCount._incompleteMPUUploads : 0;
55+
resTarget.objectCount.current += BigInt(objectCount?.current || 0n);
56+
resTarget.objectCount.nonCurrent += BigInt(objectCount?.nonCurrent || 0n);
57+
resTarget.objectCount.deleteMarker += BigInt(objectCount?.deleteMarker || 0n);
58+
resTarget.objectCount._currentCold += BigInt(objectCount?._currentCold || 0n);
59+
resTarget.objectCount._nonCurrentCold += BigInt(objectCount?._nonCurrentCold || 0n);
60+
resTarget.objectCount._currentRestoring += BigInt(objectCount?._currentRestoring || 0n);
61+
resTarget.objectCount._currentRestored += BigInt(objectCount?._currentRestored || 0n);
62+
resTarget.objectCount._nonCurrentRestoring += BigInt(objectCount?._nonCurrentRestoring || 0n);
63+
resTarget.objectCount._nonCurrentRestored += BigInt(objectCount?._nonCurrentRestored || 0n);
64+
resTarget.objectCount._incompleteMPUUploads += BigInt(objectCount?._incompleteMPUUploads || 0n);
6565

66-
resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : 0;
66+
resTarget.usedCapacity._inflightsPreScan += BigInt(usedCapacity?._inflightsPreScan || 0n);
6767
if (accountOwnerID) {
6868
resTarget.accountOwnerID = accountOwnerID;
6969
}
7070

7171
resTarget.usedCapacity.current += usedCapacity
72-
? usedCapacity._currentCold + usedCapacity._currentRestored + usedCapacity._currentRestoring
73-
+ usedCapacity._incompleteMPUParts : 0;
72+
? BigInt(usedCapacity._currentCold) + BigInt(usedCapacity._currentRestored) + BigInt(usedCapacity._currentRestoring)
73+
+ BigInt(usedCapacity._incompleteMPUParts) : 0n;
7474
resTarget.usedCapacity.nonCurrent += usedCapacity
75-
? usedCapacity._nonCurrentCold + usedCapacity._nonCurrentRestored + usedCapacity._nonCurrentRestoring : 0;
75+
? BigInt(usedCapacity._nonCurrentCold) + BigInt(usedCapacity._nonCurrentRestored) + BigInt(usedCapacity._nonCurrentRestoring) : 0n;
7676
resTarget.objectCount.current += objectCount
77-
? objectCount._currentCold + objectCount._currentRestored + objectCount._currentRestoring
78-
+ objectCount._incompleteMPUUploads : 0;
77+
? BigInt(objectCount._currentCold) + BigInt(objectCount._currentRestored) + BigInt(objectCount._currentRestoring)
78+
+ BigInt(objectCount._incompleteMPUUploads) : 0n;
7979
resTarget.objectCount.nonCurrent += objectCount
80-
? objectCount._nonCurrentCold + objectCount._nonCurrentRestored + objectCount._nonCurrentRestoring : 0;
80+
? BigInt(objectCount._nonCurrentCold) + BigInt(objectCount._nonCurrentRestored) + BigInt(objectCount._nonCurrentRestoring) : 0n;
8181

8282
return resTarget;
8383
}
8484

85+
function serializeBigInts(obj) {
86+
if (typeof obj !== 'object' || obj === null) {
87+
return typeof obj === 'bigint' ? { __bigint: obj.toString() } : obj;
88+
}
89+
const result = Array.isArray(obj) ? [] : {};
90+
for (const key in obj) {
91+
if (Object.prototype.hasOwnProperty.call(obj, key)) {
92+
result[key] = serializeBigInts(obj[key]);
93+
}
94+
}
95+
return result;
96+
}
97+
98+
function deserializeBigInts(obj) {
99+
if (!obj || typeof obj !== 'object') {
100+
return obj;
101+
}
102+
if (obj.__bigint !== undefined) {
103+
return BigInt(obj.__bigint);
104+
}
105+
const result = Array.isArray(obj) ? [] : {};
106+
for (const key in obj) {
107+
if (Object.prototype.hasOwnProperty.call(obj, key)) {
108+
result[key] = deserializeBigInts(obj[key]);
109+
}
110+
}
111+
return result;
112+
}
113+
85114
module.exports = {
86115
consolidateDataMetrics,
116+
serializeBigInts,
117+
deserializeBigInts,
87118
};

‎DataReport/collectBucketMetricsAndUpdateBucketCapacityInfo.js‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,19 @@ function isSOSCapacityInfoEnabled(bucketInfo) {
3232

3333
function isValidBucketStorageMetrics(bucketMetric) {
3434
return bucketMetric
35-
&& bucketMetric.usedCapacity
36-
&& typeof bucketMetric.usedCapacity.current === 'number'
37-
&& typeof bucketMetric.usedCapacity.nonCurrent === 'number'
35+
&& bucketMetric.usedCapacity
36+
// For backward compatibility reeasons, we accept bigint and numbers
37+
// But only bigints will be stored in the database
38+
&& (typeof bucketMetric.usedCapacity.current === 'bigint' || typeof bucketMetric.usedCapacity.current === 'number')
39+
&& (typeof bucketMetric.usedCapacity.nonCurrent === 'bigint' || typeof bucketMetric.usedCapacity.nonCurrent === 'number')
3840
&& bucketMetric.usedCapacity.current > -1
3941
&& bucketMetric.usedCapacity.nonCurrent > -1;
4042
}
4143

4244
function isValidCapacityValue(capacity) {
43-
return (Number.isSafeInteger(capacity) && capacity >= 0);
45+
// For backward compatibility reasons, we accept bigint and numbers
46+
// But only bigints (Longs) will be stored in the database
47+
return ((typeof capacity === 'bigint' || (typeof capacity === 'number' && Number.isInteger(capacity))) && capacity >= 0);
4448
}
4549

4650
function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callback) {
@@ -74,22 +78,28 @@ function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callb
7478
return nxt(null, doc);
7579
}),
7680
(storageMetricDoc, nxt) => {
77-
let bucketStorageUsed = -1;
81+
let bucketStorageUsed = -1n;
7882
if (isValidBucketStorageMetrics(storageMetricDoc)) {
7983
// Do not count the objects in cold for SOSAPI
80-
bucketStorageUsed = storageMetricDoc.usedCapacity.current
81-
+ storageMetricDoc.usedCapacity.nonCurrent;
84+
// numbers are converted into bigints to ensure we support more
85+
// than 9PB of bytes stored and previous clusters where the
86+
// values were stored as numbers.
87+
bucketStorageUsed = BigInt(storageMetricDoc.usedCapacity.current)
88+
+ BigInt(storageMetricDoc.usedCapacity.nonCurrent);
8289
}
8390
// read Capacity from bucket._capabilities
8491
const { Capacity } = bucket.getCapabilities().VeeamSOSApi.CapacityInfo;
8592

86-
let available = -1;
87-
let capacity = -1;
88-
if (isValidCapacityValue(Capacity)) { // is Capacity value is valid
93+
let available = -1n;
94+
let capacity = -1n;
95+
if (isValidCapacityValue(Capacity)) {
96+
// is Capacity value is valid
8997
capacity = Capacity;
9098
// if bucket storage used is valid and capacity is bigger than used
91-
if (bucketStorageUsed !== -1 && (capacity - bucketStorageUsed) >= 0) {
99+
if (bucketStorageUsed !== -1n && (capacity - bucketStorageUsed) >= 0n) {
92100
available = capacity - bucketStorageUsed;
101+
} else if (bucketStorageUsed !== -1n && (capacity - bucketStorageUsed) < 0n) {
102+
available = 0n;
93103
}
94104
}
95105
return mongoClient.updateBucketCapacityInfo(bucketName, {

‎bucketVersionsStats.js‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -132,12 +132,12 @@ const s3 = new AWS.S3(Object.assign(options, s3Options));
132132

133133
const stats = {
134134
current: {
135-
count: 0,
136-
size: 0,
135+
count: 0n,
136+
size: 0n,
137137
},
138138
noncurrent: {
139-
count: 0,
140-
size: 0,
139+
count: 0n,
140+
size: 0n,
141141
},
142142
};
143143

@@ -147,8 +147,8 @@ let VersionIdMarker;
147147
function _logProgress(message) {
148148
const loggedStats = {
149149
total: {
150-
count: stats.current.count + stats.noncurrent.count,
151-
size: stats.current.size + stats.noncurrent.size,
150+
count: BigInt(stats.current.count + stats.noncurrent.count),
151+
size: BigInt(stats.current.size + stats.noncurrent.size),
152152
},
153153
...stats,
154154
};
@@ -199,8 +199,8 @@ function listBucket(bucket, cb) {
199199
}
200200
}
201201
const statObj = version.IsLatest ? stats.current : stats.noncurrent;
202-
statObj.count += 1;
203-
statObj.size += version.Size || 0;
202+
statObj.count += 1n;
203+
statObj.size += version.Size || 0n;
204204
if (VERBOSE) {
205205
log.info('version info', {
206206
bucket: BUCKET,

‎package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
"dependencies": {
2929
"@senx/warp10": "^1.1.2",
3030
"JSONStream": "^1.3.5",
31-
"arsenal": "git+https://github.com/scality/arsenal#8.1.142",
31+
"arsenal": "git+https://github.com/scality/arsenal#8.1.146",
3232
"async": "^2.6.4",
3333
"aws-sdk": "^2.1005.0",
3434
"bucketclient": "git+https://github.com/scality/bucketclient#8.1.5",
@@ -56,8 +56,9 @@
5656
"eslint-config-scality": "github:scality/Guidelines#8.2.0",
5757
"eslint-plugin-import": "^2.20.1",
5858
"eslint-plugin-jest": "^23.6.0",
59-
"jest": "^23.6.0",
59+
"jest": "^29.7.0",
6060
"mongodb-memory-server": "^8.10.2",
61+
"randomatic": "^3.1.1",
6162
"sinon": "^17.0.1"
6263
}
6364
}

0 commit comments

Comments
 (0)