Skip to content

Commit f00680a

Browse files
rpmoorerpmoore
committed
Added a delete 'directory' test to the bucket integration tests. Added several data integrity tests for various blob sizes with different byte boundries
1 parent e39cb25 commit f00680a

File tree

4 files changed

+248
-0
lines changed

4 files changed

+248
-0
lines changed

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataInputStream.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.spectralogic.ds3client.integration;
2+
3+
import java.io.IOException;
4+
import java.io.InputStream;
5+
import java.util.Random;
6+
7+
public class RandomDataInputStream extends InputStream {
8+
9+
private final Random rand;
10+
private final long length;
11+
private long count = 0;
12+
13+
public RandomDataInputStream(final long seed, final long length) {
14+
this.rand = new Random(seed);
15+
this.length = length;
16+
}
17+
18+
@Override
19+
public int read() throws IOException {
20+
if (count >= length) {
21+
return -1;
22+
}
23+
24+
count++;
25+
return rand.nextInt();
26+
}
27+
}

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,4 +108,5 @@ public static void deleteAllContents(final Ds3Client client, final String bucket
108108

109109
client.deleteBucket(new DeleteBucketRequest(bucketName));
110110
}
111+
111112
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,25 @@
99

1010
import java.io.IOException;
1111
import java.net.URISyntaxException;
12+
import java.nio.ByteBuffer;
1213
import java.nio.channels.SeekableByteChannel;
1314
import java.nio.file.Files;
1415
import java.nio.file.Path;
1516
import java.nio.file.StandardOpenOption;
1617
import java.security.SignatureException;
18+
import java.util.Iterator;
19+
import java.util.List;
1720
import java.util.UUID;
1821

22+
import com.google.common.collect.Iterables;
23+
import com.google.common.collect.Lists;
1924
import com.spectralogic.ds3client.commands.*;
2025
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
26+
import com.spectralogic.ds3client.models.Contents;
27+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
2128
import com.spectralogic.ds3client.models.bulk.JobStatus;
29+
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
30+
import org.apache.commons.io.IOUtils;
2231
import org.junit.BeforeClass;
2332
import org.junit.Test;
2433

@@ -226,4 +235,49 @@ public void negativePutDuplicateObject() throws SignatureException,
226235
Util.deleteAllContents(client, bucketName);
227236
}
228237
}
238+
239+
@Test
240+
public void deleteDirectory() throws IOException, SignatureException, XmlProcessingException {
241+
final String bucketName = "delete_directory";
242+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
243+
244+
try {
245+
helpers.ensureBucketExists(bucketName);
246+
247+
final List<Ds3Object> objects = Lists.newArrayList(
248+
new Ds3Object("dirA/obj1.txt", 1024),
249+
new Ds3Object("dirA/obj2.txt", 1024),
250+
new Ds3Object("dirA/obj3.txt", 1024),
251+
new Ds3Object("obj1.txt", 1024));
252+
253+
final Ds3ClientHelpers.Job putJob = helpers.startWriteJob(bucketName, objects);
254+
255+
putJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
256+
@Override
257+
public SeekableByteChannel buildChannel(String key) throws IOException {
258+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(120, 1024));
259+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
260+
261+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(1024);
262+
channel.write(randomBuffer);
263+
264+
return channel;
265+
}
266+
});
267+
268+
final Iterable<Contents> objs = helpers.listObjects(bucketName, "dirA");
269+
270+
for (final Contents objContents : objs) {
271+
client.deleteObject(new DeleteObjectRequest(bucketName, objContents.getKey()));
272+
}
273+
274+
final Iterable<Contents> filesLeft = helpers.listObjects(bucketName);
275+
276+
assertTrue(Iterables.size(filesLeft) == 1);
277+
}
278+
finally {
279+
Util.deleteAllContents(client, bucketName);
280+
}
281+
282+
}
229283
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/DataIntegrity_Test.java‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package com.spectralogic.ds3client.integration;
2+
3+
import com.google.common.collect.Lists;
4+
import com.spectralogic.ds3client.Ds3Client;
5+
import com.spectralogic.ds3client.commands.BulkPutRequest;
6+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
7+
import com.spectralogic.ds3client.helpers.FileObjectGetter;
8+
import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
9+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
10+
import com.spectralogic.ds3client.serializer.XmlProcessingException;
11+
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
12+
import com.spectralogic.ds3client.utils.ResourceUtils;
13+
import org.apache.commons.codec.digest.DigestUtils;
14+
import org.apache.commons.io.IOUtils;
15+
import org.junit.BeforeClass;
16+
import org.junit.Test;
17+
18+
import java.io.File;
19+
import java.io.FileInputStream;
20+
import java.io.IOException;
21+
import java.net.URISyntaxException;
22+
import java.nio.ByteBuffer;
23+
import java.nio.channels.SeekableByteChannel;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
import java.security.SignatureException;
27+
28+
import static org.hamcrest.CoreMatchers.equalTo;
29+
import static org.hamcrest.CoreMatchers.is;
30+
import static org.junit.Assert.assertThat;
31+
32+
public class DataIntegrity_Test {
33+
private static Ds3Client client;
34+
35+
@BeforeClass
36+
public static void startup() {
37+
client = Util.fromEnv();
38+
}
39+
40+
@Test
41+
public void singleFilePut() throws IOException, URISyntaxException, XmlProcessingException, SignatureException {
42+
final String bucketName = "java_integration_test";
43+
final String book = "beowulf.txt";
44+
45+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
46+
47+
try {
48+
helpers.ensureBucketExists(bucketName);
49+
50+
final File objFile = ResourceUtils.loadFileResource(Util.RESOURCE_BASE_NAME + book);
51+
final String digest = DigestUtils.sha256Hex(new FileInputStream(objFile));
52+
final Ds3Object obj = new Ds3Object(book, objFile.length());
53+
54+
final Ds3ClientHelpers.Job putJob = helpers.startWriteJob(bucketName, Lists.newArrayList(obj));
55+
putJob.transfer(new ResourceObjectPutter(Util.RESOURCE_BASE_NAME));
56+
57+
final Path tempDir = Files.createTempDirectory("ds3_test_");
58+
59+
final Ds3ClientHelpers.Job getJob = helpers.startReadAllJob(bucketName);
60+
getJob.transfer(new FileObjectGetter(tempDir));
61+
62+
final String secondDigest = DigestUtils.sha256Hex(Files.newInputStream(tempDir.resolve(book)));
63+
assertThat(secondDigest, is(equalTo(digest)));
64+
}
65+
finally {
66+
Util.deleteAllContents(client, bucketName);
67+
}
68+
}
69+
70+
@Test
71+
public void randomDataFile() throws IOException, SignatureException, URISyntaxException, XmlProcessingException {
72+
final String bucketName = "java_integration_test";
73+
final String randomFileName = "random.txt";
74+
final long seed = 12345689;
75+
final int length = 2048;
76+
77+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
78+
}
79+
80+
@Test
81+
public void randomSingleByte() throws XmlProcessingException, SignatureException, IOException {
82+
final String bucketName = "java_integration_test";
83+
final String randomFileName = "random.txt";
84+
final long seed = 12345689;
85+
final int length = 1;
86+
87+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
88+
}
89+
90+
@Test
91+
public void multiBlob() throws IOException, SignatureException, XmlProcessingException {
92+
final String bucketName = "java_integration_test";
93+
final String randomFileName = "random.txt";
94+
final long seed = 12345689;
95+
final int length = 2 * BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES;
96+
97+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
98+
}
99+
100+
@Test
101+
public void fullBlobPlusOneByte() throws XmlProcessingException, SignatureException, IOException {
102+
final String bucketName = "java_integration_test";
103+
final String randomFileName = "random.txt";
104+
final long seed = 12345;
105+
final int length = BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES + 1;
106+
107+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
108+
}
109+
110+
@Test
111+
public void threeFullBlobs() throws XmlProcessingException, SignatureException, IOException {
112+
final String bucketName = "java_integration_test";
113+
final String randomFileName = "random.txt";
114+
final long seed = 12345;
115+
final int length = 3 * BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES;
116+
117+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
118+
}
119+
120+
@Test
121+
public void twoFullBlobsPlusOneByte() throws XmlProcessingException, SignatureException, IOException {
122+
final String bucketName = "java_integration_test";
123+
final String randomFileName = "random.txt";
124+
final long seed = 12345;
125+
final int length = 2 * BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES + 1;
126+
127+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
128+
}
129+
130+
public void sendAndVerifySingleFile(final String bucketName, final String fileName, final long seed, final int length) throws IOException, SignatureException, XmlProcessingException {
131+
try {
132+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
133+
helpers.ensureBucketExists(bucketName);
134+
135+
final String digest = DigestUtils.sha256Hex(new RandomDataInputStream(seed, length));
136+
final Ds3Object obj = new Ds3Object(fileName, length);
137+
138+
final Ds3ClientHelpers.Job putJob = helpers.startWriteJob(bucketName, Lists.newArrayList(obj),
139+
WriteJobOptions.create().withMaxUploadSize(BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES));
140+
putJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
141+
@Override
142+
public SeekableByteChannel buildChannel(final String key) throws IOException {
143+
144+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(seed, length));
145+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
146+
147+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(length);
148+
channel.write(randomBuffer);
149+
150+
return channel;
151+
}
152+
});
153+
154+
final Path tempDir = Files.createTempDirectory("ds3_test_");
155+
156+
final Ds3ClientHelpers.Job getJob = helpers.startReadAllJob(bucketName);
157+
getJob.transfer(new FileObjectGetter(tempDir));
158+
159+
final String secondDigest = DigestUtils.sha256Hex(Files.newInputStream(tempDir.resolve(fileName)));
160+
assertThat(secondDigest, is(equalTo(digest)));
161+
}
162+
finally {
163+
Util.deleteAllContents(client, bucketName);
164+
}
165+
}
166+
}

0 commit comments

Comments
 (0)