Skip to content

Commit fca083b

Browse files
BourgoisMickaelBourgoisMickael
committed
(wip) functional tests for sse KMS Migration
Need a script to run before version then upgrade version and rerun I'll try to clean this test
1 parent 3ec9594 commit fca083b

File tree

8 files changed

+2416
-0
lines changed

8 files changed

+2416
-0
lines changed

‎package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@
7474
"ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js",
7575
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management",
7676
"ft_kmip": "cd tests/functional/kmip && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js",
77+
"ft_sse_cleanup": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js",
78+
"ft_sse_before_migration": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js beforeMigration.js",
79+
"ft_sse_migration": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 migration.js",
80+
"ft_sse_arn": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js arnPrefix.js",
7781
"install_ft_deps": "yarn install [email protected] [email protected] [email protected] [email protected] [email protected]",
7882
"lint": "eslint $(git ls-files '*.js')",
7983
"lint_md": "mdlint $(git ls-files '*.md')",

‎tests/functional/sse-kms-migration/arnPrefix.js‎

Lines changed: 678 additions & 0 deletions
Large diffs are not rendered by default.

‎tests/functional/sse-kms-migration/beforeMigration.js‎

Lines changed: 672 additions & 0 deletions
Large diffs are not rendered by default.

‎tests/functional/sse-kms-migration/cleanup.js‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/* eslint-disable */
2+
const getConfig = require('../aws-node-sdk/test/support/config');
3+
const { S3 } = require('aws-sdk');
4+
const { promisify } = require('util');
5+
const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util');
6+
7+
const metadata = require('../../../lib/metadata/wrapper');
8+
9+
// use file to defined key in arn prefix, if no prefix mem is used
10+
11+
// copy part of aws-node-sdk/test/object/encryptionHeaders.js and add more tests
12+
// around SSE Key prefix and migration
13+
// always getObject to ensure decryption
14+
15+
const testCases = [
16+
{
17+
name: 'algo-none',
18+
// as the init insert objects with each encryption
19+
// this bucket will have a non mandatory AES256
20+
},
21+
{
22+
name: 'algo-none-del-sse',
23+
/** flag to remove non mandatory AES256 SS3 from bucket MD beforeEach test */
24+
deleteSSE: true,
25+
},
26+
{
27+
name: 'algo-aes256',
28+
algo: 'AES256',
29+
},
30+
{
31+
name: 'algo-awskms',
32+
algo: 'aws:kms',
33+
},
34+
{
35+
name: 'algo-awskms-key',
36+
algo: 'aws:kms',
37+
masterKeyId: true,
38+
},
39+
{
40+
name: 'algo-awskms-key-arnprefix',
41+
algo: 'aws:kms',
42+
masterKeyId: true,
43+
arnPrefix: true,
44+
},
45+
];
46+
47+
const config = getConfig('default', { signatureVersion: 'v4' });
48+
const s3 = new S3(config);
49+
const bucketUtil = new BucketUtility();
50+
51+
async function cleanup(Bucket) {
52+
try {
53+
void await bucketUtil.empty(Bucket);
54+
void await s3.deleteBucket({ Bucket }).promise();
55+
} catch (e) {
56+
console.log('Ignore error for', Bucket, e.toString());
57+
}
58+
}
59+
60+
describe('SSE KMS Cleanup', () => {
61+
/** Bucket to test CopyObject from and to */
62+
const copyBkt = 'enc-bkt-copy';
63+
const mpuCopyBkt = 'enc-bkt-mpu-copy';
64+
65+
it('Empty and delete buckets for SSE KMS Migration', async () => {
66+
void await promisify(metadata.setup.bind(metadata))();
67+
68+
try {
69+
// pre cleanup
70+
void await cleanup(copyBkt);
71+
void await cleanup(mpuCopyBkt);
72+
void await Promise.all(testCases.map(async bktConf => {
73+
void await cleanup(`enc-bkt-${bktConf.name}`);
74+
return await cleanup(`versioned-enc-bkt-${bktConf.name}`);
75+
}));
76+
} catch (e) { void e; }
77+
});
78+
});

‎tests/functional/sse-kms-migration/config.before.json‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
{
2+
"port": 8000,
3+
"listenOn": [],
4+
"metricsPort": 8002,
5+
"metricsListenOn": [],
6+
"replicationGroupId": "RG001",
7+
"restEndpoints": {
8+
"localhost": "us-east-1",
9+
"127.0.0.1": "us-east-1",
10+
"cloudserver-front": "us-east-1",
11+
"s3.docker.test": "us-east-1",
12+
"127.0.0.2": "us-east-1",
13+
"s3.amazonaws.com": "us-east-1"
14+
},
15+
"websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
16+
"s3-website.us-east-2.amazonaws.com",
17+
"s3-website-us-west-1.amazonaws.com",
18+
"s3-website-us-west-2.amazonaws.com",
19+
"s3-website.ap-south-1.amazonaws.com",
20+
"s3-website.ap-northeast-2.amazonaws.com",
21+
"s3-website-ap-southeast-1.amazonaws.com",
22+
"s3-website-ap-southeast-2.amazonaws.com",
23+
"s3-website-ap-northeast-1.amazonaws.com",
24+
"s3-website.eu-central-1.amazonaws.com",
25+
"s3-website-eu-west-1.amazonaws.com",
26+
"s3-website-sa-east-1.amazonaws.com",
27+
"s3-website.localhost",
28+
"s3-website.scality.test"],
29+
"replicationEndpoints": [{
30+
"site": "zenko",
31+
"servers": ["127.0.0.1:8000"],
32+
"default": true
33+
}, {
34+
"site": "us-east-2",
35+
"type": "aws_s3"
36+
}],
37+
"cdmi": {
38+
"host": "localhost",
39+
"port": 81,
40+
"path": "/dewpoint",
41+
"readonly": true
42+
},
43+
"bucketd": {
44+
"bootstrap": ["localhost:9000"]
45+
},
46+
"vaultd": {
47+
"host": "localhost",
48+
"port": 8500
49+
},
50+
"clusters": 10,
51+
"log": {
52+
"logLevel": "info",
53+
"dumpLevel": "error"
54+
},
55+
"healthChecks": {
56+
"allowFrom": ["127.0.0.1/8", "::1"]
57+
},
58+
"metadataClient": {
59+
"host": "localhost",
60+
"port": 9990
61+
},
62+
"dataClient": {
63+
"host": "localhost",
64+
"port": 9991
65+
},
66+
"metadataDaemon": {
67+
"bindAddress": "localhost",
68+
"port": 9990
69+
},
70+
"dataDaemon": {
71+
"bindAddress": "localhost",
72+
"port": 9991
73+
},
74+
"recordLog": {
75+
"enabled": false,
76+
"recordLogName": "s3-recordlog"
77+
},
78+
"requests": {
79+
"viaProxy": false,
80+
"trustedProxyCIDRs": [],
81+
"extractClientIPFromHeader": ""
82+
},
83+
"bucketNotificationDestinations": [
84+
{
85+
"resource": "target1",
86+
"type": "dummy",
87+
"host": "localhost:6000"
88+
}
89+
],
90+
"defaultEncryptionKeyPerAccount": true
91+
}

‎tests/functional/sse-kms-migration/config.sseMigration.hideArn.json‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"port": 8000,
3+
"listenOn": [],
4+
"metricsPort": 8002,
5+
"metricsListenOn": [],
6+
"replicationGroupId": "RG001",
7+
"restEndpoints": {
8+
"localhost": "us-east-1",
9+
"127.0.0.1": "us-east-1",
10+
"cloudserver-front": "us-east-1",
11+
"s3.docker.test": "us-east-1",
12+
"127.0.0.2": "us-east-1",
13+
"s3.amazonaws.com": "us-east-1"
14+
},
15+
"websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
16+
"s3-website.us-east-2.amazonaws.com",
17+
"s3-website-us-west-1.amazonaws.com",
18+
"s3-website-us-west-2.amazonaws.com",
19+
"s3-website.ap-south-1.amazonaws.com",
20+
"s3-website.ap-northeast-2.amazonaws.com",
21+
"s3-website-ap-southeast-1.amazonaws.com",
22+
"s3-website-ap-southeast-2.amazonaws.com",
23+
"s3-website-ap-northeast-1.amazonaws.com",
24+
"s3-website.eu-central-1.amazonaws.com",
25+
"s3-website-eu-west-1.amazonaws.com",
26+
"s3-website-sa-east-1.amazonaws.com",
27+
"s3-website.localhost",
28+
"s3-website.scality.test"],
29+
"replicationEndpoints": [{
30+
"site": "zenko",
31+
"servers": ["127.0.0.1:8000"],
32+
"default": true
33+
}, {
34+
"site": "us-east-2",
35+
"type": "aws_s3"
36+
}],
37+
"cdmi": {
38+
"host": "localhost",
39+
"port": 81,
40+
"path": "/dewpoint",
41+
"readonly": true
42+
},
43+
"bucketd": {
44+
"bootstrap": ["localhost:9000"]
45+
},
46+
"vaultd": {
47+
"host": "localhost",
48+
"port": 8500
49+
},
50+
"clusters": 10,
51+
"log": {
52+
"logLevel": "info",
53+
"dumpLevel": "error"
54+
},
55+
"healthChecks": {
56+
"allowFrom": ["127.0.0.1/8", "::1"]
57+
},
58+
"metadataClient": {
59+
"host": "localhost",
60+
"port": 9990
61+
},
62+
"dataClient": {
63+
"host": "localhost",
64+
"port": 9991
65+
},
66+
"metadataDaemon": {
67+
"bindAddress": "localhost",
68+
"port": 9990
69+
},
70+
"dataDaemon": {
71+
"bindAddress": "localhost",
72+
"port": 9991
73+
},
74+
"recordLog": {
75+
"enabled": false,
76+
"recordLogName": "s3-recordlog"
77+
},
78+
"requests": {
79+
"viaProxy": false,
80+
"trustedProxyCIDRs": [],
81+
"extractClientIPFromHeader": ""
82+
},
83+
"bucketNotificationDestinations": [
84+
{
85+
"resource": "target1",
86+
"type": "dummy",
87+
"host": "localhost:6000"
88+
}
89+
],
90+
"defaultEncryptionKeyPerAccount": true,
91+
"kmsHideScalityArn": true,
92+
"sseMigration": {
93+
"previousKeyType": "internal",
94+
"previousKeyProtocol": "file",
95+
"previousKeyProvider": "scality"
96+
}
97+
}

‎tests/functional/sse-kms-migration/config.sseMigration.showArn.json‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"port": 8000,
3+
"listenOn": [],
4+
"metricsPort": 8002,
5+
"metricsListenOn": [],
6+
"replicationGroupId": "RG001",
7+
"restEndpoints": {
8+
"localhost": "us-east-1",
9+
"127.0.0.1": "us-east-1",
10+
"cloudserver-front": "us-east-1",
11+
"s3.docker.test": "us-east-1",
12+
"127.0.0.2": "us-east-1",
13+
"s3.amazonaws.com": "us-east-1"
14+
},
15+
"websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
16+
"s3-website.us-east-2.amazonaws.com",
17+
"s3-website-us-west-1.amazonaws.com",
18+
"s3-website-us-west-2.amazonaws.com",
19+
"s3-website.ap-south-1.amazonaws.com",
20+
"s3-website.ap-northeast-2.amazonaws.com",
21+
"s3-website-ap-southeast-1.amazonaws.com",
22+
"s3-website-ap-southeast-2.amazonaws.com",
23+
"s3-website-ap-northeast-1.amazonaws.com",
24+
"s3-website.eu-central-1.amazonaws.com",
25+
"s3-website-eu-west-1.amazonaws.com",
26+
"s3-website-sa-east-1.amazonaws.com",
27+
"s3-website.localhost",
28+
"s3-website.scality.test"],
29+
"replicationEndpoints": [{
30+
"site": "zenko",
31+
"servers": ["127.0.0.1:8000"],
32+
"default": true
33+
}, {
34+
"site": "us-east-2",
35+
"type": "aws_s3"
36+
}],
37+
"cdmi": {
38+
"host": "localhost",
39+
"port": 81,
40+
"path": "/dewpoint",
41+
"readonly": true
42+
},
43+
"bucketd": {
44+
"bootstrap": ["localhost:9000"]
45+
},
46+
"vaultd": {
47+
"host": "localhost",
48+
"port": 8500
49+
},
50+
"clusters": 10,
51+
"log": {
52+
"logLevel": "info",
53+
"dumpLevel": "error"
54+
},
55+
"healthChecks": {
56+
"allowFrom": ["127.0.0.1/8", "::1"]
57+
},
58+
"metadataClient": {
59+
"host": "localhost",
60+
"port": 9990
61+
},
62+
"dataClient": {
63+
"host": "localhost",
64+
"port": 9991
65+
},
66+
"metadataDaemon": {
67+
"bindAddress": "localhost",
68+
"port": 9990
69+
},
70+
"dataDaemon": {
71+
"bindAddress": "localhost",
72+
"port": 9991
73+
},
74+
"recordLog": {
75+
"enabled": false,
76+
"recordLogName": "s3-recordlog"
77+
},
78+
"requests": {
79+
"viaProxy": false,
80+
"trustedProxyCIDRs": [],
81+
"extractClientIPFromHeader": ""
82+
},
83+
"bucketNotificationDestinations": [
84+
{
85+
"resource": "target1",
86+
"type": "dummy",
87+
"host": "localhost:6000"
88+
}
89+
],
90+
"defaultEncryptionKeyPerAccount": true,
91+
"kmsHideScalityArn": false,
92+
"sseMigration": {
93+
"previousKeyType": "internal",
94+
"previousKeyProtocol": "file",
95+
"previousKeyProvider": "scality"
96+
}
97+
}

0 commit comments

Comments
 (0)