Skip to content

Commit 825a9df

Browse files
benzekrimahabenzekrimaha
committed
bump arsenal
1 parent 1322253 commit 825a9df

File tree

21 files changed

+673
-98
lines changed

21 files changed

+673
-98
lines changed

‎package.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
"@azure/storage-blob": "^12.28.0",
2929
"@hapi/joi": "^17.1.1",
3030
"@smithy/node-http-handler": "^3.0.0",
31-
"arsenal": "git+https://github.com/scality/Arsenal#8.2.35",
31+
"arsenal": "git+https://github.com/scality/Arsenal#28666f91719dec9e8380731d9d61dce09bf0b410",
3232
"async": "2.6.4",
3333
"bucketclient": "scality/bucketclient#8.2.7",
3434
"bufferutil": "^4.0.8",
@@ -106,8 +106,7 @@
106106
"ft_s3cmd": "cd tests/functional/s3cmd && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
107107
"ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
108108
"ft_util": "cd tests/functional/utilities && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
109-
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util ft_backbeat",
110-
"ft_search": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/mdSearch --exit",
109+
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util ft_backbeat", "ft_search": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/mdSearch --exit",
111110
"ft_kmip": "cd tests/functional/kmip && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
112111
"ft_kmip_cluster": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 300000 load.js --exit",
113112
"ft_sse_cleanup": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js --exit",

‎tests/functional/aws-node-sdk/test/support/awsConfig.js‎

Lines changed: 70 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
const { fromIni } = require('@aws-sdk/credential-providers');
21
const fs = require('fs');
32
const path = require('path');
43
const { config } = require('../../../../../lib/Config');
@@ -15,7 +14,26 @@ function getAwsCredentials(profile, credFile = '/.aws/credentials') {
1514
throw new Error(msg);
1615
}
1716

18-
return fromIni({ profile, filepath: filename });
17+
// Parse the INI file manually for synchronous access
18+
// SDK v3's fromIni is async, but arsenal's GCP client needs sync access to credentials
19+
const content = fs.readFileSync(filename, 'utf-8');
20+
const profileMatch = content.match(new RegExp(`\\[${profile}\\][\\s\\S]*?(?=\\n\\[|$)`));
21+
22+
if (!profileMatch) {
23+
throw new Error(`Profile "${profile}" not found in ${filename}`);
24+
}
25+
26+
const accessKeyMatch = profileMatch[0].match(/aws_access_key_id\s*=\s*(.+)/);
27+
const secretKeyMatch = profileMatch[0].match(/aws_secret_access_key\s*=\s*(.+)/);
28+
29+
if (!accessKeyMatch || !secretKeyMatch) {
30+
throw new Error(`Missing credentials in profile "${profile}"`);
31+
}
32+
33+
return {
34+
accessKeyId: accessKeyMatch[1].trim(),
35+
secretAccessKey: secretKeyMatch[1].trim(),
36+
};
1937
}
2038

2139
function getRealAwsConfig(location) {
@@ -24,15 +42,20 @@ function getRealAwsConfig(location) {
2442
config.locationConstraints[location].details;
2543
const useHTTPS = config.locationConstraints[location].details.https;
2644
const proto = useHTTPS ? 'https' : 'http';
45+
const isGcp = config.locationConstraints[location].type === 'gcp';
2746
const params = {
2847
region: 'us-east-1',
2948
endpoint: gcpEndpoint ?
3049
`${proto}://${gcpEndpoint}` : `${proto}://${awsEndpoint}`,
3150
};
32-
if (config.locationConstraints[location].type === 'gcp') {
51+
52+
if (isGcp) {
53+
params.disableS3ExpressSessionAuth = true;
54+
params.useGlobalEndpoint = false;
55+
params.s3DisableBodySigning = true;
3356
params.mainBucket = bucketName;
3457
params.mpuBucket = mpuBucketName;
35-
}
58+
}
3659
if (useHTTPS) {
3760
params.requestHandler = {
3861
httpsAgent: new https.Agent({ keepAlive: true }),
@@ -42,19 +65,56 @@ function getRealAwsConfig(location) {
4265
httpAgent: new http.Agent({ keepAlive: true }),
4366
};
4467
}
68+
69+
if (pathStyle) {
70+
params.forcePathStyle = true;
71+
}
72+
73+
if (!useHTTPS) {
74+
params.sslEnabled = false;
75+
}
76+
4577
if (credentialsProfile) {
46-
const credentials = getAwsCredentials(credentialsProfile,
47-
'/.aws/credentials');
78+
const credentials = getAwsCredentials(credentialsProfile, '/.aws/credentials');
4879
params.credentials = credentials;
80+
81+
if (isGcp) {
82+
return {
83+
s3Params: params,
84+
bucketName,
85+
mpuBucket: mpuBucketName || bucketName,
86+
credentials: { // For raw HTTP requests (GCP format)
87+
accessKey: credentials.accessKeyId,
88+
secretKey: credentials.secretAccessKey,
89+
},
90+
};
91+
}
4992
return params;
5093
}
51-
if (pathStyle) {
52-
params.forcePathStyle = true;
53-
}
54-
params.credentials = {
94+
params.credentials = {
5595
accessKeyId: locCredentials.accessKey,
5696
secretAccessKey: locCredentials.secretKey,
5797
};
98+
99+
// For GCP with plain credentials, return nested structure
100+
if (isGcp) {
101+
return {
102+
s3Params: {
103+
...params,
104+
credentials: {
105+
accessKeyId: locCredentials.accessKey,
106+
secretAccessKey: locCredentials.secretKey,
107+
},
108+
},
109+
bucketName,
110+
mpuBucket: mpuBucketName || bucketName,
111+
credentials: {
112+
accessKey: locCredentials.accessKey,
113+
secretKey: locCredentials.secretKey,
114+
},
115+
};
116+
}
117+
58118
return params;
59119
}
60120

‎tests/functional/raw-node/test/GCP/bucket/get.js‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
const assert = require('assert');
22
const async = require('async');
33
const arsenal = require('arsenal');
4-
const { GCP } = arsenal.storage.data.external;
4+
const { GCP } = arsenal.storage.data.external.GCP;
5+
56
const { makeGcpRequest } = require('../../../utils/makeRequest');
67
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
78
const { getRealAwsConfig } =
@@ -25,11 +26,13 @@ function populateBucket(createdObjects, callback) {
2526
bucket: bucketName,
2627
objectKey: object,
2728
authCredentials: config.credentials,
28-
}, err => moveOn(err));
29+
}, err => {
30+
moveOn(err);
31+
});
2932
}, err => {
3033
if (err) {
3134
process.stdout
32-
.write(`err putting objects ${err.code}`);
35+
.write(`err putting objects ${err.code}\n`);
3336
}
3437
return callback(err);
3538
});
@@ -49,7 +52,7 @@ function removeObjects(createdObjects, callback) {
4952
}, err => {
5053
if (err) {
5154
process.stdout
52-
.write(`err deleting objects ${err.code}`);
55+
.write(`err deleting objects ${err.code}\n`);
5356
}
5457
return callback(err);
5558
});
@@ -87,12 +90,22 @@ describe('GCP: GET Bucket', function testSuite() {
8790
describe('without existing bucket', () => {
8891
it('should return 404 and NoSuchBucket', done => {
8992
const badBucketName = `nonexistingbucket-${genUniqID()}`;
90-
gcpClient.getBucket({
93+
gcpClient.listObjects({
9194
Bucket: badBucketName,
9295
}, err => {
9396
assert(err);
94-
assert.strictEqual(err.statusCode, 404);
95-
assert.strictEqual(err.code, 'NoSuchBucket');
97+
assert.strictEqual(err.$metadata?.httpStatusCode, 404);
98+
assert.strictEqual(err.name, 'NoSuchBucket');
99+
return done();
100+
});
101+
});
102+
103+
it('should return 200', done => {
104+
gcpClient.listObjects({
105+
Bucket: bucketName,
106+
}, (err, res) => {
107+
assert.equal(err, null, `Expected success, but got ${err}`);
108+
assert.strictEqual(res.$metadata?.httpStatusCode, 200);
96109
return done();
97110
});
98111
});

‎tests/functional/raw-node/test/GCP/bucket/getVersioning.js‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
const assert = require('assert');
22
const async = require('async');
33
const arsenal = require('arsenal');
4-
const { GCP } = arsenal.storage.data.external;
4+
const { GCP } = arsenal.storage.data.external.GCP;
55
const { makeGcpRequest } = require('../../../utils/makeRequest');
66
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
77
const { getRealAwsConfig } =
88
require('../../../../aws-node-sdk/test/support/awsConfig');
99

1010
const credentialOne = 'gcpbackend';
11-
const verEnabledObj = { Status: 'Enabled' };
12-
const verDisabledObj = { Status: 'Suspended' };
11+
const verEnabledObj = 'Enabled';
12+
const verDisabledObj = 'Suspended';
1313
const xmlEnable =
1414
'<?xml version="1.0" encoding="UTF-8"?>' +
1515
'<VersioningConfiguration>' +
@@ -33,7 +33,7 @@ describe('GCP: GET Bucket Versioning', () => {
3333
authCredentials: config.credentials,
3434
}, 0, err => {
3535
if (err) {
36-
process.stdout.write(`err in creating bucket ${err}\n`);
36+
process.stdout.write(`err in creating bucket ${err.code}\n`);
3737
}
3838
return done(err);
3939
});
@@ -46,7 +46,7 @@ describe('GCP: GET Bucket Versioning', () => {
4646
authCredentials: config.credentials,
4747
}, 0, err => {
4848
if (err) {
49-
process.stdout.write(`err in deleting bucket ${err}\n`);
49+
process.stdout.write(`err in deleting bucket ${err.code}\n`);
5050
}
5151
return done(err);
5252
});
@@ -58,22 +58,24 @@ describe('GCP: GET Bucket Versioning', () => {
5858
method: 'PUT',
5959
bucket: this.test.bucketName,
6060
authCredentials: config.credentials,
61-
queryObj: { versioning: {} },
61+
queryObj: { versioning: '' },
6262
requestBody: xmlEnable,
6363
}, err => {
6464
if (err) {
65-
process.stdout.write(`err in setting versioning ${err}`);
65+
process.stdout.write(`err in setting versioning ${err.code}`);
6666
}
6767
return next(err);
6868
}),
69-
next => gcpClient.getBucketVersioning({
70-
Bucket: this.test.bucketName,
71-
}, (err, res) => {
72-
assert.equal(err, null,
73-
`Expected success, but got err ${err}`);
74-
assert.deepStrictEqual(res, verEnabledObj);
75-
return next();
76-
}),
69+
next => {
70+
gcpClient.getBucketVersioning({
71+
Bucket: this.test.bucketName,
72+
}, (err, res) => {
73+
assert.equal(err, null,
74+
`Expected success, but got err ${err}`);
75+
assert.deepStrictEqual(res.Status, verEnabledObj);
76+
return next();
77+
});
78+
},
7779
], err => done(err));
7880
});
7981

@@ -83,7 +85,7 @@ describe('GCP: GET Bucket Versioning', () => {
8385
method: 'PUT',
8486
bucket: this.test.bucketName,
8587
authCredentials: config.credentials,
86-
queryObj: { versioning: {} },
88+
queryObj: { versioning: '' },
8789
requestBody: xmlDisable,
8890
}, err => {
8991
if (err) {
@@ -96,7 +98,7 @@ describe('GCP: GET Bucket Versioning', () => {
9698
}, (err, res) => {
9799
assert.equal(err, null,
98100
`Expected success, but got err ${err}`);
99-
assert.deepStrictEqual(res, verDisabledObj);
101+
assert.deepStrictEqual(res.Status, verDisabledObj);
100102
return next();
101103
}),
102104
], err => done(err));

‎tests/functional/raw-node/test/GCP/bucket/head.js‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
const assert = require('assert');
22
const arsenal = require('arsenal');
3-
const { GCP } = arsenal.storage.data.external;
3+
const { GCP } = arsenal.storage.data.external.GCP;
44
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
55
const { getRealAwsConfig } =
66
require('../../../../aws-node-sdk/test/support/awsConfig');
@@ -22,7 +22,7 @@ describe('GCP: HEAD Bucket', () => {
2222
Bucket: this.test.bucketName,
2323
}, err => {
2424
assert(err);
25-
assert.strictEqual(err.statusCode, 404);
25+
assert.strictEqual(err.$metadata?.httpStatusCode, 404);
2626
return done();
2727
});
2828
});
@@ -67,7 +67,9 @@ describe('GCP: HEAD Bucket', () => {
6767
Bucket: this.test.bucketName,
6868
}, (err, res) => {
6969
assert.equal(err, null, `Expected success, but got ${err}`);
70-
assert.deepStrictEqual(this.test.bucketObj, res);
70+
const { $metadata, ...data } = res;
71+
assert.strictEqual($metadata.httpStatusCode, 200);
72+
assert.deepStrictEqual(this.test.bucketObj, data);
7173
return done();
7274
});
7375
});

‎tests/functional/raw-node/test/GCP/bucket/putVersioning.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ const assert = require('assert');
22
const async = require('async');
33
const arsenal = require('arsenal');
44
const xml2js = require('xml2js');
5-
const { GCP } = arsenal.storage.data.external;
5+
const { GCP } = arsenal.storage.data.external.GCP;
66
const { makeGcpRequest } = require('../../../utils/makeRequest');
77
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
88
const { getRealAwsConfig } =
@@ -70,7 +70,7 @@ describe('GCP: PUT Bucket Versioning', () => {
7070
method: 'GET',
7171
bucket: this.test.bucketName,
7272
authCredentials: config.credentials,
73-
queryObj: { versioning: {} },
73+
queryObj: { versioning: '' },
7474
}, (err, res) => {
7575
if (err) {
7676
process.stdout.write(`err in retrieving bucket ${err}`);
@@ -97,7 +97,7 @@ describe('GCP: PUT Bucket Versioning', () => {
9797
method: 'GET',
9898
bucket: this.test.bucketName,
9999
authCredentials: config.credentials,
100-
queryObj: { versioning: {} },
100+
queryObj: { versioning: '' },
101101
}, (err, res) => {
102102
if (err) {
103103
process.stdout.write(`err in retrieving bucket ${err}`);

‎tests/functional/raw-node/test/GCP/object/completeMpu.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const assert = require('assert');
22
const async = require('async');
33
const arsenal = require('arsenal');
4-
const { GCP, GcpUtils } = arsenal.storage.data.external;
4+
const { GCP, GcpUtils } = arsenal.storage.data.external.GCP;
55
const { gcpRequestRetry, setBucketClass, gcpMpuSetup, genUniqID } =
66
require('../../../utils/gcpUtils');
77
const { getRealAwsConfig } =

‎tests/functional/raw-node/test/GCP/object/copy.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const assert = require('assert');
22
const async = require('async');
33
const arsenal = require('arsenal');
4-
const { GCP } = arsenal.storage.data.external;
4+
const { GCP } = arsenal.storage.data.external.GCP;
55
const { makeGcpRequest } = require('../../../utils/makeRequest');
66
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
77
const { getRealAwsConfig } =
@@ -51,8 +51,8 @@ describe('GCP: COPY Object', function testSuite() {
5151
CopySource: `/${bucketName}/${missingObject}`,
5252
}, err => {
5353
assert(err);
54-
assert.strictEqual(err.statusCode, 404);
55-
assert.strictEqual(err.code, 'NoSuchKey');
54+
assert.strictEqual(err.$metadata.httpStatusCode, 404);
55+
assert.strictEqual(err.name, 'NoSuchKey');
5656
return done();
5757
});
5858
});

‎tests/functional/raw-node/test/GCP/object/delete.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const assert = require('assert');
22
const async = require('async');
33
const arsenal = require('arsenal');
4-
const { GCP } = arsenal.storage.data.external;
4+
const { GCP } = arsenal.storage.data.external.GCP;
55
const { makeGcpRequest } = require('../../../utils/makeRequest');
66
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
77
const { getRealAwsConfig } =
@@ -90,8 +90,8 @@ describe('GCP: DELETE Object', function testSuite() {
9090
Key: badObjectKey,
9191
}, err => {
9292
assert(err);
93-
assert.strictEqual(err.statusCode, 404);
94-
assert.strictEqual(err.code, 'NoSuchKey');
93+
assert.strictEqual(err.$metadata.httpStatusCode, 404);
94+
assert.strictEqual(err.name, 'NoSuchKey');
9595
return done();
9696
});
9797
});

0 commit comments

Comments
 (0)