Skip to content

Commit 283b5e0

Browse files
tcarmettcarmet
committed
S3UTILS-211: ft tests for listObjectsByReplicationStatus
1 parent b4a493f commit 283b5e0

File tree

1 file changed

+369
-0
lines changed

1 file changed

+369
-0
lines changed

‎tests/functional/listObjectsByReplicationStatus.js‎

Lines changed: 369 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,369 @@
1+
const vaultclient = require('vaultclient');
2+
const { promisify } = require('util');
3+
const AWS = require('aws-sdk');
4+
const { Logger } = require('werelogs');
5+
const { listObjectsByReplicationStatus } = require('../../listObjectsByReplicationStatus');
6+
const admincredentials = require('../../node_modules/vaultclient/tests/utils/admincredentials.json');
7+
8+
const log = new Logger('listObjectsByReplicationStatus:test');
9+
10+
const iamHost = process.env.IAM_HOST || 'localhost';
11+
const iamPort = process.env.IAM_PORT || 8600;
12+
const s3Host = process.env.S3_HOST || 'localhost';
13+
const s3Port = process.env.S3_PORT || 8000;
14+
15+
const adminAccessKeyId = process.env.ADMIN_ACCESS_KEY_ID || Object.keys(admincredentials)[0];
16+
const adminSecretAccessKey = process.env.ADMIN_SECRET_ACCESS_KEY || admincredentials[adminAccessKeyId];
17+
const region = 'us-east-1';
18+
19+
describe('listObjectsByReplicationStatus', () => {
20+
let vaultClient;
21+
let accountSource;
22+
let accountDest;
23+
24+
beforeAll(async () => {
25+
vaultClient = new vaultclient.Client(
26+
iamHost,
27+
iamPort,
28+
false,
29+
undefined,
30+
undefined,
31+
undefined,
32+
undefined,
33+
adminAccessKeyId,
34+
adminSecretAccessKey
35+
);
36+
});
37+
38+
beforeEach(async () => {
39+
log.info('Setting up test accounts and buckets');
40+
accountSource = await createTestAccount(vaultClient);
41+
accountDest = await createTestAccount(vaultClient);
42+
log.info(`Account source: ${accountSource.accountName} and dest: ${accountDest.accountName} were created`);
43+
log.info('Configuring CRR between source and destination buckets');
44+
await configureCrr(accountSource, accountDest);
45+
});
46+
47+
afterEach(async () => {
48+
log.info('Cleaning up test accounts');
49+
await removeCrrConfiguration(accountSource);
50+
await removeCrrConfiguration(accountDest);
51+
await deleteTestAccount(vaultClient, accountSource);
52+
await deleteTestAccount(vaultClient, accountDest);
53+
log.info('Test accounts deleted');
54+
});
55+
56+
it('should list objects by replication status', async () => {
57+
// Add data to source bucket
58+
log.info('Uploading test objects to source bucket');
59+
const testObjects = [
60+
{ Key: 'test-object-1', Body: 'data to replicate 1' },
61+
{ Key: 'test-object-2', Body: 'data to replicate 2' },
62+
{ Key: 'test-object-3', Body: 'data to replicate 3' },
63+
];
64+
65+
for (const obj of testObjects) {
66+
await accountSource.s3Client.putObject({
67+
Bucket: accountSource.bucketName,
68+
Key: obj.Key,
69+
Body: obj.Body,
70+
}).promise();
71+
log.info('Uploaded object', { key: obj.Key });
72+
}
73+
74+
// Verify objects have replication status
75+
log.info('Verifying objects have replication status');
76+
for (const obj of testObjects) {
77+
const headResult = await accountSource.s3Client.headObject({
78+
Bucket: accountSource.bucketName,
79+
Key: obj.Key,
80+
}).promise();
81+
log.info('Object metadata', {
82+
key: obj.Key,
83+
replicationStatus: headResult.ReplicationStatus,
84+
versionId: headResult.VersionId
85+
});
86+
}
87+
88+
// Execute the listObjectsByReplicationStatus function directly
89+
log.info('Executing listObjectsByReplicationStatus function');
90+
const endpoint = `http://${s3Host}:${s3Port}`;
91+
92+
// Call the function directly for coverage
93+
await listObjectsByReplicationStatus({
94+
buckets: accountSource.bucketName,
95+
accessKey: accountSource.accountAccessKey,
96+
secretKey: accountSource.accountSecretKey,
97+
endpoint,
98+
replicationStatus: 'PENDING,FAILED,COMPLETED',
99+
logger: log,
100+
});
101+
102+
log.info('Function execution completed successfully');
103+
}, 30000); // Increase timeout for script execution
104+
});
105+
106+
async function configureCrr(accountSource, accountDest) {
107+
// activate bucket versionning on source and destination buckets
108+
log.info('Enabling bucket versioning on source and destination buckets');
109+
await accountSource.s3Client.putBucketVersioning({
110+
Bucket: accountSource.bucketName,
111+
VersioningConfiguration: {
112+
Status: 'Enabled',
113+
},
114+
}).promise();
115+
116+
await accountDest.s3Client.putBucketVersioning({
117+
Bucket: accountDest.bucketName,
118+
VersioningConfiguration: {
119+
Status: 'Enabled',
120+
},
121+
}).promise();
122+
123+
log.info('Creating IAM policies and roles for CRR');
124+
// create policy
125+
const policy = {
126+
Version:'2012-10-17',
127+
Statement:[
128+
{
129+
Effect:'Allow',
130+
Action:[
131+
's3:GetObjectVersion',
132+
's3:GetObjectVersionAcl',
133+
's3:ReplicateObject'
134+
],
135+
Resource:[
136+
`arn:aws:s3:::${accountSource.bucketName}/*`
137+
]
138+
},
139+
{
140+
Effect:'Allow',
141+
Action:[
142+
's3:ListBucket',
143+
's3:GetReplicationConfiguration'
144+
],
145+
Resource:[
146+
'arn:aws:s3:::source'
147+
]
148+
},
149+
{
150+
Effect:'Allow',
151+
Action:[
152+
's3:ReplicateObject',
153+
's3:ReplicateDelete'
154+
],
155+
Resource:`arn:aws:s3:::${accountDest.bucketName}/*`
156+
}
157+
]
158+
};
159+
await accountSource.iamClient.createPolicy({
160+
PolicyName: 'crr-policy',
161+
PolicyDocument: JSON.stringify(policy),
162+
}).promise();
163+
164+
await accountDest.iamClient.createPolicy({
165+
PolicyName: 'crr-policy',
166+
PolicyDocument: JSON.stringify(policy),
167+
}).promise();
168+
169+
log.info('Creating IAM roles');
170+
// create trust
171+
const trust = {
172+
Version:'2012-10-17',
173+
Statement:[
174+
{
175+
Effect:'Allow',
176+
Principal:{
177+
Service:'backbeat'
178+
},
179+
Action:'sts:AssumeRole'
180+
}
181+
]
182+
};
183+
await accountSource.iamClient.createRole({
184+
RoleName: 'crr-trust-role',
185+
AssumeRolePolicyDocument: JSON.stringify(trust),
186+
}).promise();
187+
await accountDest.iamClient.createRole({
188+
RoleName: 'crr-trust-role',
189+
AssumeRolePolicyDocument: JSON.stringify(trust),
190+
}).promise();
191+
192+
log.info('Attaching policies to roles');
193+
// attach role to policy
194+
await accountSource.iamClient.attachRolePolicy({
195+
RoleName: 'crr-trust-role',
196+
PolicyArn: `arn:aws:iam::${accountSource.account.account.id}:policy/crr-policy`,
197+
}).promise();
198+
await accountDest.iamClient.attachRolePolicy({
199+
RoleName: 'crr-trust-role',
200+
PolicyArn: `arn:aws:iam::${accountDest.account.account.id}:policy/crr-policy`,
201+
}).promise();
202+
203+
log.info('Setting bucket replication configuration on source bucket');
204+
const replication = {
205+
Role: `arn:aws:iam::${accountSource.account.account.id}:role/crr-trust-role,arn:aws:iam::${accountDest.account.account.id}:role/crr-trust-role`,
206+
Rules: [
207+
{
208+
Prefix: '',
209+
Status: 'Enabled',
210+
Destination: {
211+
Bucket: `arn:aws:s3:::${accountDest.bucketName}`
212+
}
213+
}
214+
]
215+
};
216+
await accountSource.s3Client.putBucketReplication({
217+
Bucket: accountSource.bucketName,
218+
ReplicationConfiguration: replication
219+
}).promise();
220+
221+
}
222+
223+
async function createTestAccount(vaultClient) {
224+
const iamEndpoint = new AWS.Endpoint(`http://${iamHost}:${iamPort}`);
225+
const s3Endpoint = new AWS.Endpoint(`http://${s3Host}:${s3Port}`);
226+
const accountName = `test-account-${Math.random().toString(36).substring(2, 8)}`;
227+
const accountEmail = `${accountName}@example.com`;
228+
const bucketName = accountName;
229+
const iamUser = `${accountName}-user`;
230+
// Create account with vaultclient
231+
const account = await promisify(vaultClient.createAccount.bind(vaultClient))(
232+
accountName, { email: accountEmail });
233+
// Create account access key
234+
const credentials = await promisify(vaultClient.generateAccountAccessKey.bind(
235+
vaultClient))(accountName);
236+
const accountAccessKey = credentials.id;
237+
const accountSecretKey = credentials.value;
238+
239+
const iamClient = new AWS.IAM({
240+
credentials: {
241+
accessKeyId: accountAccessKey,
242+
secretAccessKey: accountSecretKey
243+
},
244+
endpoint: iamEndpoint.href,
245+
region,
246+
});
247+
248+
const s3Client = new AWS.S3({
249+
accessKeyId: accountAccessKey,
250+
secretAccessKey: accountSecretKey,
251+
region,
252+
endpoint: s3Endpoint.href
253+
});
254+
255+
await s3Client.createBucket({ Bucket: bucketName }).promise();
256+
await iamClient.createUser({ UserName: iamUser }).promise();
257+
258+
return {
259+
accountName,
260+
accountEmail,
261+
account,
262+
bucketName,
263+
accountAccessKey,
264+
accountSecretKey,
265+
iamUser,
266+
iamClient,
267+
s3Client,
268+
};
269+
270+
}
271+
272+
async function removeCrrConfiguration(account) {
273+
log.info('Removing bucket crr configuration', { bucket: account.bucketName });
274+
try {
275+
await account.s3Client.deleteBucketReplication({ Bucket: account.bucketName }).promise();
276+
} catch (err) {
277+
log.error('Error removing bucket replication configuration', {
278+
bucket: account.bucketName,
279+
error: err.message,
280+
});
281+
}
282+
// remove versioning
283+
await account.s3Client.putBucketVersioning({
284+
Bucket: account.bucketName,
285+
VersioningConfiguration: {
286+
Status: 'Suspended',
287+
},
288+
}).promise();
289+
290+
// Clean up IAM resources first (roles and policies must be deleted before account)
291+
log.info('Cleaning up IAM resources', { account: account.accountName });
292+
try {
293+
// Detach policy from role
294+
await account.iamClient.detachRolePolicy({
295+
RoleName: 'crr-trust-role',
296+
PolicyArn: `arn:aws:iam::${account.account.account.id}:policy/crr-policy`,
297+
}).promise();
298+
log.info('Detached policy from role');
299+
} catch (err) {
300+
log.error('Error detaching policy from role', { error: err.message });
301+
}
302+
303+
try {
304+
// Delete role
305+
await account.iamClient.deleteRole({ RoleName: 'crr-trust-role' }).promise();
306+
log.info('Deleted IAM role');
307+
} catch (err) {
308+
log.error('Error deleting role', { error: err.message });
309+
}
310+
311+
try {
312+
// Delete policy
313+
await account.iamClient.deletePolicy({
314+
PolicyArn: `arn:aws:iam::${account.account.account.id}:policy/crr-policy`,
315+
}).promise();
316+
log.info('Deleted IAM policy');
317+
} catch (err) {
318+
log.error('Error deleting policy', { error: err.message });
319+
}
320+
321+
try {
322+
// Delete IAM user
323+
await account.iamClient.deleteUser({ UserName: account.iamUser }).promise();
324+
log.info('Deleted IAM user');
325+
} catch (err) {
326+
log.error('Error deleting IAM user', { error: err.message });
327+
}
328+
}
329+
330+
async function deleteTestAccount(vaultClient, account) {
331+
// Delete bucket
332+
log.info('Deleting bucket', { bucket: account.bucketName });
333+
// empty bucket - need to delete all versions and delete markers for versioned buckets
334+
const listedObjects = await account.s3Client.listObjectVersions({ Bucket: account.bucketName }).promise();
335+
336+
const objectsToDelete = [];
337+
338+
// Add all object versions
339+
if (listedObjects.Versions && listedObjects.Versions.length > 0) {
340+
listedObjects.Versions.forEach(({ Key, VersionId }) => {
341+
log.info('Scheduling object version for deletion', { key: Key, versionId: VersionId });
342+
objectsToDelete.push({ Key, VersionId });
343+
});
344+
}
345+
346+
// Add all delete markers
347+
if (listedObjects.DeleteMarkers && listedObjects.DeleteMarkers.length > 0) {
348+
listedObjects.DeleteMarkers.forEach(({ Key, VersionId }) => {
349+
log.info('Scheduling delete marker for deletion', { key: Key, versionId: VersionId });
350+
objectsToDelete.push({ Key, VersionId });
351+
});
352+
}
353+
354+
// Delete all versions and markers
355+
if (objectsToDelete.length > 0) {
356+
const deleteParams = {
357+
Bucket: account.bucketName,
358+
Delete: { Objects: objectsToDelete }
359+
};
360+
await account.s3Client.deleteObjects(deleteParams).promise();
361+
}
362+
363+
await account.s3Client.deleteBucket({ Bucket: account.bucketName }).promise();
364+
log.info('Deleted bucket', { bucket: account.bucketName });
365+
366+
// Delete account with vaultclient
367+
await promisify(vaultClient.deleteAccount.bind(vaultClient))(account.accountName);
368+
log.info('Deleted account', { account: account.accountName });
369+
}

0 commit comments

Comments
 (0)