Skip to content

Commit 396e044

Browse files
author
hansdude
committed
Merge pull request #63 from rpmoore/master
Data Integrity Testing
2 parents f8ece35 + 075154e commit 396e044

File tree

10 files changed

+285
-3
lines changed

10 files changed

+285
-3
lines changed

‎ds3-sdk-integration/build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
dependencies {
2+
compile 'commons-codec:commons-codec:1.10'
3+
}

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataInputStream.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.spectralogic.ds3client.integration;
2+
3+
import java.io.IOException;
4+
import java.io.InputStream;
5+
import java.util.Random;
6+
7+
public class RandomDataInputStream extends InputStream {
8+
9+
private final Random rand;
10+
private final long length;
11+
private long count = 0;
12+
13+
public RandomDataInputStream(final long seed, final long length) {
14+
this.rand = new Random(seed);
15+
this.length = length;
16+
}
17+
18+
@Override
19+
public int read() throws IOException {
20+
if (count >= length) {
21+
return -1;
22+
}
23+
24+
count++;
25+
return rand.nextInt();
26+
}
27+
}

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,4 +108,5 @@ public static void deleteAllContents(final Ds3Client client, final String bucket
108108

109109
client.deleteBucket(new DeleteBucketRequest(bucketName));
110110
}
111+
111112
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,25 @@
99

1010
import java.io.IOException;
1111
import java.net.URISyntaxException;
12+
import java.nio.ByteBuffer;
1213
import java.nio.channels.SeekableByteChannel;
1314
import java.nio.file.Files;
1415
import java.nio.file.Path;
1516
import java.nio.file.StandardOpenOption;
1617
import java.security.SignatureException;
18+
import java.util.Iterator;
19+
import java.util.List;
1720
import java.util.UUID;
1821

22+
import com.google.common.collect.Iterables;
23+
import com.google.common.collect.Lists;
1924
import com.spectralogic.ds3client.commands.*;
2025
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
26+
import com.spectralogic.ds3client.models.Contents;
27+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
2128
import com.spectralogic.ds3client.models.bulk.JobStatus;
29+
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
30+
import org.apache.commons.io.IOUtils;
2231
import org.junit.BeforeClass;
2332
import org.junit.Test;
2433

@@ -226,4 +235,49 @@ public void negativePutDuplicateObject() throws SignatureException,
226235
Util.deleteAllContents(client, bucketName);
227236
}
228237
}
238+
239+
@Test
240+
public void deleteDirectory() throws IOException, SignatureException, XmlProcessingException {
241+
final String bucketName = "delete_directory";
242+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
243+
244+
try {
245+
helpers.ensureBucketExists(bucketName);
246+
247+
final List<Ds3Object> objects = Lists.newArrayList(
248+
new Ds3Object("dirA/obj1.txt", 1024),
249+
new Ds3Object("dirA/obj2.txt", 1024),
250+
new Ds3Object("dirA/obj3.txt", 1024),
251+
new Ds3Object("obj1.txt", 1024));
252+
253+
final Ds3ClientHelpers.Job putJob = helpers.startWriteJob(bucketName, objects);
254+
255+
putJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
256+
@Override
257+
public SeekableByteChannel buildChannel(String key) throws IOException {
258+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(120, 1024));
259+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
260+
261+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(1024);
262+
channel.write(randomBuffer);
263+
264+
return channel;
265+
}
266+
});
267+
268+
final Iterable<Contents> objs = helpers.listObjects(bucketName, "dirA");
269+
270+
for (final Contents objContents : objs) {
271+
client.deleteObject(new DeleteObjectRequest(bucketName, objContents.getKey()));
272+
}
273+
274+
final Iterable<Contents> filesLeft = helpers.listObjects(bucketName);
275+
276+
assertTrue(Iterables.size(filesLeft) == 1);
277+
}
278+
finally {
279+
Util.deleteAllContents(client, bucketName);
280+
}
281+
282+
}
229283
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/DataIntegrity_Test.java‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package com.spectralogic.ds3client.integration;
2+
3+
import com.google.common.collect.Lists;
4+
import com.spectralogic.ds3client.Ds3Client;
5+
import com.spectralogic.ds3client.commands.BulkPutRequest;
6+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
7+
import com.spectralogic.ds3client.helpers.FileObjectGetter;
8+
import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
9+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
10+
import com.spectralogic.ds3client.serializer.XmlProcessingException;
11+
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
12+
import com.spectralogic.ds3client.utils.ResourceUtils;
13+
import org.apache.commons.codec.digest.DigestUtils;
14+
import org.apache.commons.io.IOUtils;
15+
import org.junit.BeforeClass;
16+
import org.junit.Test;
17+
18+
import java.io.File;
19+
import java.io.FileInputStream;
20+
import java.io.IOException;
21+
import java.net.URISyntaxException;
22+
import java.nio.ByteBuffer;
23+
import java.nio.channels.SeekableByteChannel;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
import java.security.SignatureException;
27+
28+
import static org.hamcrest.CoreMatchers.equalTo;
29+
import static org.hamcrest.CoreMatchers.is;
30+
import static org.junit.Assert.assertThat;
31+
32+
public class DataIntegrity_Test {
33+
private static Ds3Client client;
34+
35+
@BeforeClass
36+
public static void startup() {
37+
client = Util.fromEnv();
38+
}
39+
40+
@Test
41+
public void singleFilePut() throws IOException, URISyntaxException, XmlProcessingException, SignatureException {
42+
final String bucketName = "java_integration_test";
43+
final String book = "beowulf.txt";
44+
45+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
46+
47+
try {
48+
helpers.ensureBucketExists(bucketName);
49+
50+
final File objFile = ResourceUtils.loadFileResource(Util.RESOURCE_BASE_NAME + book);
51+
final String digest = DigestUtils.sha256Hex(new FileInputStream(objFile));
52+
final Ds3Object obj = new Ds3Object(book, objFile.length());
53+
54+
final Ds3ClientHelpers.Job putJob = helpers.startWriteJob(bucketName, Lists.newArrayList(obj));
55+
putJob.transfer(new ResourceObjectPutter(Util.RESOURCE_BASE_NAME));
56+
57+
final Path tempDir = Files.createTempDirectory("ds3_test_");
58+
59+
final Ds3ClientHelpers.Job getJob = helpers.startReadAllJob(bucketName);
60+
getJob.transfer(new FileObjectGetter(tempDir));
61+
62+
final String secondDigest = DigestUtils.sha256Hex(Files.newInputStream(tempDir.resolve(book)));
63+
assertThat(secondDigest, is(equalTo(digest)));
64+
}
65+
finally {
66+
Util.deleteAllContents(client, bucketName);
67+
}
68+
}
69+
70+
@Test
71+
public void randomDataFile() throws IOException, SignatureException, URISyntaxException, XmlProcessingException {
72+
final String bucketName = "java_integration_test";
73+
final String randomFileName = "random.txt";
74+
final long seed = 12345689;
75+
final int length = 2048;
76+
77+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
78+
}
79+
80+
@Test
81+
public void randomSingleByte() throws XmlProcessingException, SignatureException, IOException {
82+
final String bucketName = "java_integration_test";
83+
final String randomFileName = "random.txt";
84+
final long seed = 12345689;
85+
final int length = 1;
86+
87+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
88+
}
89+
90+
@Test
91+
public void multiBlob() throws IOException, SignatureException, XmlProcessingException {
92+
final String bucketName = "java_integration_test";
93+
final String randomFileName = "random.txt";
94+
final long seed = 12345689;
95+
final int length = 2 * BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES;
96+
97+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
98+
}
99+
100+
@Test
101+
public void fullBlobPlusOneByte() throws XmlProcessingException, SignatureException, IOException {
102+
final String bucketName = "java_integration_test";
103+
final String randomFileName = "random.txt";
104+
final long seed = 12345;
105+
final int length = BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES + 1;
106+
107+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
108+
}
109+
110+
@Test
111+
public void threeFullBlobs() throws XmlProcessingException, SignatureException, IOException {
112+
final String bucketName = "java_integration_test";
113+
final String randomFileName = "random.txt";
114+
final long seed = 12345;
115+
final int length = 3 * BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES;
116+
117+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
118+
}
119+
120+
@Test
121+
public void twoFullBlobsPlusOneByte() throws XmlProcessingException, SignatureException, IOException {
122+
final String bucketName = "java_integration_test";
123+
final String randomFileName = "random.txt";
124+
final long seed = 12345;
125+
final int length = 2 * BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES + 1;
126+
127+
sendAndVerifySingleFile(bucketName, randomFileName, seed, length);
128+
}
129+
130+
public void sendAndVerifySingleFile(final String bucketName, final String fileName, final long seed, final int length) throws IOException, SignatureException, XmlProcessingException {
131+
try {
132+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
133+
helpers.ensureBucketExists(bucketName);
134+
135+
final String digest = DigestUtils.sha256Hex(new RandomDataInputStream(seed, length));
136+
final Ds3Object obj = new Ds3Object(fileName, length);
137+
138+
final Ds3ClientHelpers.Job putJob = helpers.startWriteJob(bucketName, Lists.newArrayList(obj),
139+
WriteJobOptions.create().withMaxUploadSize(BulkPutRequest.MIN_UPLOAD_SIZE_IN_BYTES));
140+
putJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
141+
@Override
142+
public SeekableByteChannel buildChannel(final String key) throws IOException {
143+
144+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(seed, length));
145+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
146+
147+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(length);
148+
channel.write(randomBuffer);
149+
150+
return channel;
151+
}
152+
});
153+
154+
final Path tempDir = Files.createTempDirectory("ds3_test_");
155+
156+
final Ds3ClientHelpers.Job getJob = helpers.startReadAllJob(bucketName);
157+
getJob.transfer(new FileObjectGetter(tempDir));
158+
159+
final String secondDigest = DigestUtils.sha256Hex(Files.newInputStream(tempDir.resolve(fileName)));
160+
assertThat(secondDigest, is(equalTo(digest)));
161+
}
162+
finally {
163+
Util.deleteAllContents(client, bucketName);
164+
}
165+
}
166+
}

‎ds3-sdk/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ artifacts {
4040

4141
dependencies {
4242
compile 'org.apache.httpcomponents:httpclient:4.3.2'
43-
compile 'commons-codec:commons-codec:1.8'
43+
compile 'commons-codec:commons-codec:1.10'
4444
compile 'commons-io:commons-io:2.1'
4545
compile 'org.codehaus.woodstox:woodstox-core-asl:4.2.0'
4646
compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.5.0'

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/BulkPutRequest.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,13 @@
2424
import java.util.List;
2525

2626
public class BulkPutRequest extends BulkRequest {
27+
28+
private static final String MAX_UPLOAD_SIZE_IN_BYTES = "100000000000";
29+
public static final int MIN_UPLOAD_SIZE_IN_BYTES = 10485760;
30+
2731
public BulkPutRequest(final String bucket, final List<Ds3Object> objects) throws XmlProcessingException {
2832
super(bucket, objects);
29-
getQueryParams().put("operation", "start_bulk_put");
33+
this.getQueryParams().put("operation", "start_bulk_put");
3034
}
3135

3236
@Override
@@ -35,6 +39,21 @@ public BulkPutRequest withPriority(final Priority priority) {
3539
return this;
3640
}
3741

42+
/**
43+
* Sets the chunk size for this job.
44+
* @param size The chunk size in bytes. If the value passed in is less than MIN_UPLOAD_SIZE_IN_BYTES, then
45+
* the default size will be used.
46+
*/
47+
public BulkPutRequest withMaxUploadSize(final int size) {
48+
if (size > MIN_UPLOAD_SIZE_IN_BYTES) {
49+
this.getQueryParams().put("max_upload_size", Long.toString(size));
50+
}
51+
else {
52+
this.getQueryParams().put("max_upload_size", MAX_UPLOAD_SIZE_IN_BYTES);
53+
}
54+
return this;
55+
}
56+
3857
@Override
3958
public BulkPutRequest withWriteOptimization(final WriteOptimization writeOptimization) {
4059
super.withWriteOptimization(writeOptimization);

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket,
6969
throws SignatureException, IOException, XmlProcessingException {
7070
final BulkPutResponse prime = this.client.bulkPut(new BulkPutRequest(bucket, Lists.newArrayList(objectsToWrite))
7171
.withPriority(options.getPriority())
72-
.withWriteOptimization(options.getWriteOptimization()));
72+
.withWriteOptimization(options.getWriteOptimization())
73+
.withMaxUploadSize(options.getMaxUploadSize()));
7374
return new WriteJobImpl(this.client, prime.getResult());
7475
}
7576

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/options/WriteJobOptions.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,27 @@
2121
public class WriteJobOptions {
2222
private Priority priority;
2323
private WriteOptimization writeOptimization;
24+
private int maxUploadSize;
2425

2526
private WriteJobOptions() {
2627
this.priority = null;
2728
this.writeOptimization = null;
29+
this.maxUploadSize = 0;
2830
}
2931

3032
public static WriteJobOptions create() {
3133
return new WriteJobOptions();
3234
}
3335

36+
public WriteJobOptions withMaxUploadSize(final int maxUploadSize) {
37+
this.maxUploadSize = maxUploadSize;
38+
return this;
39+
}
40+
41+
public int getMaxUploadSize() {
42+
return this.maxUploadSize;
43+
}
44+
3445
public WriteJobOptions withWriteOptimization(final WriteOptimization writeOptimization) {
3546
this.writeOptimization = writeOptimization;
3647
return this;

‎ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/ByteArraySeekableByteChannel.java‎ renamed to ‎ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/ByteArraySeekableByteChannel.java‎

File renamed without changes.

0 commit comments

Comments
 (0)