Skip to content

Commit 6a756fb

Browse files
GraciesPadreGraciesPadre
authored
Merge pull request #542 from RachelTucker/5_0_autogen
5.0 API
2 parents 2a1ec2b + 6131765 commit 6a756fb

File tree

70 files changed

+1146
-342
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+1146
-342
lines changed

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataChannelBuilder.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.integration;
17+
18+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
19+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
20+
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
21+
import org.apache.commons.io.IOUtils;
22+
23+
import java.io.IOException;
24+
import java.nio.ByteBuffer;
25+
import java.nio.channels.SeekableByteChannel;
26+
import java.util.HashMap;
27+
import java.util.List;
28+
import java.util.Map;
29+
30+
/**
31+
* Creates a channel builder that generates random data. All object keys and sizes need to be specified.
32+
*/
33+
public class RandomDataChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
34+
35+
private static final int seed = 12345;
36+
private final Map<String, Long> objectMap = new HashMap();
37+
38+
public RandomDataChannelBuilder() {
39+
}
40+
41+
public RandomDataChannelBuilder(final List<Ds3Object> objects) {
42+
for (final Ds3Object obj : objects) {
43+
this.objectMap.put(obj.getName(), obj.getSize());
44+
}
45+
}
46+
47+
RandomDataChannelBuilder withObject(final String key, final Long size) {
48+
this.objectMap.put(key, size);
49+
return this;
50+
}
51+
52+
@Override
53+
public SeekableByteChannel buildChannel(final String key) throws IOException {
54+
if (!this.objectMap.containsKey(key)) {
55+
throw new IllegalArgumentException(String.format("Object with name '%s' was not defined for this channel builder.", key));
56+
}
57+
58+
final Long size = this.objectMap.get(key);
59+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(seed, size));
60+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
61+
62+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(size.intValue());
63+
channel.write(randomBuffer);
64+
65+
return channel;
66+
}
67+
}

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,30 @@
1515

1616
package com.spectralogic.ds3client.integration;
1717

18+
import com.google.common.collect.ImmutableList;
1819
import com.spectralogic.ds3client.Ds3Client;
1920
import com.spectralogic.ds3client.Ds3ClientBuilder;
20-
import com.spectralogic.ds3client.commands.DeleteBucketRequest;
21-
import com.spectralogic.ds3client.commands.DeleteObjectRequest;
21+
import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
22+
import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Request;
23+
import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Response;
2224
import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
2325
import com.spectralogic.ds3client.helpers.DeleteBucket;
2426
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
2527
import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
26-
import com.spectralogic.ds3client.models.Contents;
28+
import com.spectralogic.ds3client.models.Job;
29+
import com.spectralogic.ds3client.models.JobStatus;
2730
import com.spectralogic.ds3client.models.bulk.Ds3Object;
2831
import com.spectralogic.ds3client.utils.ResourceUtils;
2932
import org.slf4j.Logger;
3033
import org.slf4j.LoggerFactory;
3134

3235
import java.io.IOException;
3336
import java.net.URISyntaxException;
37+
import java.nio.channels.FileChannel;
38+
import java.nio.channels.SeekableByteChannel;
3439
import java.nio.file.Files;
3540
import java.nio.file.Path;
41+
import java.nio.file.StandardOpenOption;
3642
import java.util.ArrayList;
3743
import java.util.List;
3844

@@ -79,6 +85,19 @@ public static void loadBookTestData(final Ds3Client client, final String bucketN
7985
LOG.info("Finished loading test data...");
8086
}
8187

88+
/**
89+
* Loads a single test book to a BP with the specified object name.
90+
*/
91+
public static void loadTestBook(final Ds3Client client, final String fileName, final String objectName, final String bucketName) throws IOException, URISyntaxException {
92+
final Path filePath = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + fileName);
93+
final Ds3Object obj = new Ds3Object(objectName, Files.size(filePath));
94+
95+
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
96+
97+
final Ds3ClientHelpers.Job job = helpers.startWriteJob(bucketName, ImmutableList.of(obj));
98+
job.transfer(key -> FileChannel.open(filePath, StandardOpenOption.READ));
99+
}
100+
82101
public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final String bucketName, final String resourceBaseName) throws IOException, URISyntaxException {
83102
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
84103

@@ -118,4 +137,19 @@ public static void deleteBucketContents(final Ds3Client client, final String buc
118137
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
119138
DeleteBucket.INSTANCE.deleteBucketContents(helpers, bucketName);
120139
}
140+
141+
// Cancels all in-progress jobs associated with a bucket
142+
public static void cancelAllJobsForBucket(final Ds3Client client, final String bucketName) {
143+
try {
144+
final GetJobsSpectraS3Response getJobs = client.getJobsSpectraS3(new GetJobsSpectraS3Request().withBucketId(bucketName));
145+
146+
for (final Job job : getJobs.getJobListResult().getJobs()) {
147+
if (job.getStatus() == JobStatus.IN_PROGRESS) {
148+
client.cancelJobSpectraS3(new CancelJobSpectraS3Request(job.getJobId()));
149+
}
150+
}
151+
} catch (final IOException e) {
152+
LOG.debug("Could not cancel jobs for bucket '%s': %s", bucketName, e.getMessage());
153+
}
154+
}
121155
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575

7676
import static com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request.MIN_UPLOAD_SIZE_IN_BYTES;
7777
import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
78+
import static com.spectralogic.ds3client.integration.Util.cancelAllJobsForBucket;
7879
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
7980
import static org.hamcrest.Matchers.*;
8081
import static org.junit.Assert.*;
@@ -192,6 +193,7 @@ public void getActiveJobs() throws IOException, URISyntaxException {
192193
}
193194
assertThat(activeJobsUUIDs, contains(jobID));
194195
} finally {
196+
cancelAllJobsForBucket(client, BUCKET_NAME);
195197
deleteAllContents(client, BUCKET_NAME);
196198
}
197199
}
@@ -210,6 +212,7 @@ public void getJobs() throws IOException {
210212
assertThat(jobUUIDs, contains(jobID));
211213

212214
} finally {
215+
cancelAllJobsForBucket(client, BUCKET_NAME);
213216
deleteAllContents(client, BUCKET_NAME);
214217
}
215218
}
@@ -230,6 +233,7 @@ public void modifyJobPriority() throws IOException {
230233
assertThat(response.getMasterObjectListResult().getPriority(), is(Priority.HIGH));
231234

232235
} finally {
236+
cancelAllJobsForBucket(client, BUCKET_NAME);
233237
deleteAllContents(client, BUCKET_NAME);
234238
}
235239
}
@@ -250,6 +254,7 @@ public void modifyJobName() throws IOException {
250254
assertThat(response.getMasterObjectListResult().getName(), is("newName"));
251255

252256
} finally {
257+
cancelAllJobsForBucket(client, BUCKET_NAME);
253258
deleteAllContents(client, BUCKET_NAME);
254259
}
255260
}
@@ -275,6 +280,7 @@ public void modifyJobCreationDate() throws IOException {
275280
assertThat(responseAfterModify.getMasterObjectListResult().getStartDate(), is(newDate));
276281

277282
} finally {
283+
cancelAllJobsForBucket(client, BUCKET_NAME);
278284
deleteAllContents(client, BUCKET_NAME);
279285
}
280286
}
@@ -543,6 +549,7 @@ public void getJobChunksReady() throws IOException {
543549
assertThat(chunkNames, contains(ds3Object.getName()));
544550

545551
} finally {
552+
cancelAllJobsForBucket(client, BUCKET_NAME);
546553
deleteAllContents(client, BUCKET_NAME);
547554
}
548555
}
@@ -568,6 +575,7 @@ public void getJobChunk() throws IOException {
568575
assertThat(getJobChunkSpectraS3Response.getObjectsResult(), is(notNullValue()));
569576

570577
} finally {
578+
cancelAllJobsForBucket(client, BUCKET_NAME);
571579
deleteAllContents(client, BUCKET_NAME);
572580
}
573581
}
@@ -589,6 +597,7 @@ public void aggregateTwoJobs() throws IOException {
589597
assertThat(jobOneId, is(jobTwoId));
590598

591599
} finally {
600+
cancelAllJobsForBucket(client, BUCKET_NAME);
592601
deleteAllContents(client, BUCKET_NAME);
593602
}
594603
}
@@ -612,6 +621,7 @@ public void testPutJobOptionsWithStreamingBehavior() throws IOException {
612621
assertThat(jobOneId, is(jobTwoId));
613622

614623
} finally {
624+
cancelAllJobsForBucket(client, BUCKET_NAME);
615625
deleteAllContents(client, BUCKET_NAME);
616626
}
617627
}
@@ -635,6 +645,7 @@ public void testPutJobOptionsWithRandomAccessBehavior() throws IOException {
635645
assertThat(jobOneId, is(jobTwoId));
636646

637647
} finally {
648+
cancelAllJobsForBucket(client, BUCKET_NAME);
638649
deleteAllContents(client, BUCKET_NAME);
639650
}
640651
}
@@ -652,6 +663,7 @@ public void allocateJobChunk() throws IOException {
652663
assertThat(allocateResponse.getStatus(), is(AllocateJobChunkSpectraS3Response.Status.ALLOCATED));
653664

654665
} finally {
666+
cancelAllJobsForBucket(client, BUCKET_NAME);
655667
deleteAllContents(client, BUCKET_NAME);
656668
}
657669
}
@@ -811,6 +823,7 @@ public void initiateMultipartUpload() throws IOException {
811823
assertThat(e.getStatusCode(), is(400));
812824

813825
} finally {
826+
cancelAllJobsForBucket(client, BUCKET_NAME);
814827
deleteAllContents(client, BUCKET_NAME);
815828
}
816829
}
@@ -1217,6 +1230,7 @@ public void onFailure(final FailureEvent failureEvent) {
12171230
}
12181231
} finally {
12191232
FileUtils.deleteDirectory(tempDirectory.toFile());
1233+
cancelAllJobsForBucket(client, BUCKET_NAME);
12201234
deleteAllContents(client, BUCKET_NAME);
12211235
}
12221236
}
@@ -1290,6 +1304,7 @@ public void waiting(final int secondsToWait) {
12901304
assertEquals(maxNumObjectTransferAttempts, numWaitingForChunksEventsFired.get());
12911305
} finally {
12921306
FileUtils.deleteDirectory(tempDirectory.toFile());
1307+
cancelAllJobsForBucket(client, BUCKET_NAME);
12931308
deleteAllContents(client, BUCKET_NAME);
12941309
}
12951310
}
@@ -1324,6 +1339,7 @@ public void onFailure(final FailureEvent failureEvent) {
13241339
}
13251340
} finally {
13261341
FileUtils.deleteDirectory(tempDirectory.toFile());
1342+
cancelAllJobsForBucket(client, BUCKET_NAME);
13271343
deleteAllContents(client, BUCKET_NAME);
13281344
}
13291345
}
@@ -1366,6 +1382,7 @@ public void testPutJobUsingTransferStrategy() throws IOException, InterruptedExc
13661382
assertEquals(FILE_NAMES[0], bucketContents.getKey());
13671383
}
13681384
} finally {
1385+
cancelAllJobsForBucket(client, BUCKET_NAME);
13691386
deleteAllContents(client, BUCKET_NAME);
13701387
}
13711388
}
@@ -1411,6 +1428,7 @@ public void testWriteJobUsingTransferStrategy() throws IOException, InterruptedE
14111428
assertEquals(FILE_NAMES[0], bucketContents.getKey());
14121429
}
14131430
} finally {
1431+
cancelAllJobsForBucket(client, BUCKET_NAME);
14141432
deleteAllContents(client, BUCKET_NAME);
14151433
}
14161434
}
@@ -1473,6 +1491,7 @@ public void monitor() {
14731491

14741492
assertEquals(1, numChunkAllocationAttempts.get());
14751493
} finally {
1494+
cancelAllJobsForBucket(client, BUCKET_NAME);
14761495
deleteAllContents(client, BUCKET_NAME);
14771496
}
14781497
}
@@ -1575,6 +1594,7 @@ public void released() {
15751594
assertEquals(1, numTimesChannelOpened.get());
15761595
assertEquals(1, numTimesChannelClosed.get());
15771596
} finally {
1597+
cancelAllJobsForBucket(client, BUCKET_NAME);
15781598
deleteAllContents(client, BUCKET_NAME);
15791599
}
15801600
}
@@ -1682,6 +1702,7 @@ public void monitor() {
16821702

16831703
assertEquals(1, numTimesTransferCalled.get());
16841704
} finally {
1705+
cancelAllJobsForBucket(client, BUCKET_NAME);
16851706
deleteAllContents(client, BUCKET_NAME);
16861707
}
16871708
}
@@ -1764,6 +1785,7 @@ public void reset() {
17641785
assertEquals(0, numTimesInvokeCalled.get());
17651786
assertEquals(1, numTimesResetCalled.get());
17661787
} finally {
1788+
cancelAllJobsForBucket(client, BUCKET_NAME);
17671789
deleteAllContents(client, BUCKET_NAME);
17681790
}
17691791
}
@@ -1824,6 +1846,7 @@ public void testPutJobUsingStreamedTransferStrategy() throws IOException, URISyn
18241846
assertEquals(FILE_NAMES[0], bucketContents.getKey());
18251847
}
18261848
} finally {
1849+
cancelAllJobsForBucket(client, BUCKET_NAME);
18271850
deleteAllContents(client, BUCKET_NAME);
18281851
}
18291852
}
@@ -1857,6 +1880,7 @@ public void testPutJobUsingRandomAccessTransferStrategy() throws IOException, UR
18571880
assertEquals(FILE_NAMES[0], bucketContents.getKey());
18581881
}
18591882
} finally {
1883+
cancelAllJobsForBucket(client, BUCKET_NAME);
18601884
deleteAllContents(client, BUCKET_NAME);
18611885
}
18621886
}
@@ -1973,6 +1997,7 @@ public void testPutting15000Files() throws IOException, URISyntaxException {
19731997
fail("This test makes sure that we don't run out of connections when transferring lots of small files. Oops");
19741998
} finally {
19751999
FileUtils.deleteDirectory(tempDirectory.toFile());
2000+
cancelAllJobsForBucket(client, BUCKET_NAME);
19762001
deleteAllContents(client, BUCKET_NAME);
19772002
}
19782003
}
@@ -2006,6 +2031,7 @@ public void testThatFifoIsNotProcessed() throws IOException, InterruptedExceptio
20062031
caughtException.set(true);
20072032
} finally {
20082033
FileUtils.deleteDirectory(tempDirectory.toFile());
2034+
cancelAllJobsForBucket(client, BUCKET_NAME);
20092035
deleteAllContents(client, BUCKET_NAME);
20102036
}
20112037

@@ -2058,6 +2084,7 @@ public void testThatNonExistentFileDoesNotStopPutJob() throws IOException {
20582084
} finally {
20592085
FileUtils.deleteDirectory(tempDirectoryForFilesToGet.toFile());
20602086
FileUtils.deleteDirectory(tempDirectoryForFilesToPut.toFile());
2087+
cancelAllJobsForBucket(client, BUCKET_NAME);
20612088
deleteAllContents(client, BUCKET_NAME);
20622089
}
20632090

@@ -2124,6 +2151,7 @@ public void testStreamedPutJobWithBlobbedFile() throws Exception {
21242151

21252152
assertTrue(FileUtils.contentEquals(new File(movedFileName), new File(originalFileName)));
21262153
} finally {
2154+
cancelAllJobsForBucket(client, BUCKET_NAME);
21272155
deleteAllContents(client, BUCKET_NAME);
21282156

21292157
Files.deleteIfExists(Paths.get(originalFileName));

0 commit comments

Comments
 (0)