Skip to content

Commit c5b5550

Browse files
GraciesPadreGraciesPadre
committed
Adding a helper method to check for sufficient file space for customers to use instead of catching an exception when a read job fails.
1 parent 43572f6 commit c5b5550

File tree

8 files changed

+708
-36
lines changed

8 files changed

+708
-36
lines changed

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java‎

Lines changed: 331 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,331 @@
1+
/*
2+
* ****************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.helpers;
17+
18+
import com.spectralogic.ds3client.Ds3Client;
19+
import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
20+
import com.spectralogic.ds3client.integration.Util;
21+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
22+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
23+
import com.spectralogic.ds3client.models.ChecksumType;
24+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
25+
import com.spectralogic.ds3client.utils.ResourceUtils;
26+
import org.apache.commons.io.FileUtils;
27+
import org.junit.AfterClass;
28+
import org.junit.Before;
29+
import org.junit.BeforeClass;
30+
import org.junit.Test;
31+
import java.io.IOException;
32+
import java.net.URISyntaxException;
33+
import java.nio.file.Files;
34+
import java.nio.file.Path;
35+
import java.nio.file.Paths;
36+
import java.util.ArrayList;
37+
import java.util.Arrays;
38+
import java.util.List;
39+
import java.util.UUID;
40+
import java.util.concurrent.atomic.AtomicInteger;
41+
import java.util.concurrent.atomic.AtomicLong;
42+
43+
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
44+
45+
import static org.junit.Assert.assertEquals;
46+
import static org.junit.Assert.assertTrue;
47+
import static org.junit.Assert.assertFalse;
48+
import static org.junit.Assert.assertNull;
49+
import static org.junit.Assert.assertNotNull;
50+
51+
public class FileSystemHelper_Test {
52+
private static final Ds3Client client = Util.fromEnv();
53+
private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client);
54+
private static final String BUCKET_NAME = "File_System_Helper_Test";
55+
private static final String TEST_ENV_NAME = "FileSystem_Helper_Test";
56+
private static TempStorageIds envStorageIds;
57+
private static UUID envDataPolicyId;
58+
59+
@BeforeClass
60+
public static void startup() throws IOException {
61+
envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client);
62+
envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, client);
63+
}
64+
65+
@Before
66+
public void setupBucket() throws IOException {
67+
HELPERS.ensureBucketExists(BUCKET_NAME, envDataPolicyId);
68+
}
69+
70+
@AfterClass
71+
public static void teardown() throws IOException {
72+
TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, client);
73+
client.close();
74+
}
75+
76+
@Test
77+
public void testObjectsFitBucketThatHasContent() throws IOException, URISyntaxException {
78+
try {
79+
final String DIR_NAME = "largeFiles/";
80+
final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"};
81+
82+
final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME);
83+
84+
final AtomicLong totalBookSizes = new AtomicLong(0);
85+
86+
final List<String> bookTitles = new ArrayList<>();
87+
final List<Ds3Object> objects = new ArrayList<>();
88+
for (final String book : FILE_NAMES) {
89+
final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book);
90+
final long bookSize = Files.size(objPath);
91+
totalBookSizes.getAndAdd(bookSize);
92+
final Ds3Object obj = new Ds3Object(book, bookSize);
93+
94+
bookTitles.add(book);
95+
objects.add(obj);
96+
}
97+
98+
final int maxNumBlockAllocationRetries = 1;
99+
final int maxNumObjectTransferAttempts = 1;
100+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
101+
maxNumBlockAllocationRetries,
102+
maxNumObjectTransferAttempts);
103+
104+
final AtomicInteger numTimesCallbackCalled = new AtomicInteger(0);
105+
106+
final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
107+
writeJob.attachObjectCompletedListener(new ObjectCompletedListener() {
108+
@Override
109+
public void objectCompleted(final String name) {
110+
numTimesCallbackCalled.getAndIncrement();
111+
112+
final ObjectStorageSpaceVerificationResult objectStorageSpaceVerificationResult =
113+
ds3ClientHelpers.objectsFromBucketWillFitInDirectory(BUCKET_NAME,
114+
Arrays.asList(FILE_NAMES),
115+
Paths.get("."));
116+
117+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.OK, objectStorageSpaceVerificationResult.getVerificationStatus());
118+
assertEquals(objectStorageSpaceVerificationResult.getRequiredSpace(), totalBookSizes.get());
119+
assertTrue(objectStorageSpaceVerificationResult.getAvailableSpace() > 0);
120+
assertTrue(objectStorageSpaceVerificationResult.containsSufficientSpace());
121+
assertNull(objectStorageSpaceVerificationResult.getIoException());
122+
}
123+
});
124+
125+
writeJob.transfer(new FileObjectPutter(dirPath));
126+
127+
assertEquals(1, numTimesCallbackCalled.get());
128+
} finally {
129+
deleteAllContents(client, BUCKET_NAME);
130+
}
131+
}
132+
133+
@Test
134+
public void testObjectsFitBucketWithNonExistentBucket() {
135+
final int maxNumBlockAllocationRetries = 1;
136+
final int maxNumObjectTransferAttempts = 1;
137+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
138+
maxNumBlockAllocationRetries,
139+
maxNumObjectTransferAttempts);
140+
141+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
142+
"bad bucket name", Arrays.asList(new String[] {}), Paths.get("."));
143+
144+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.BucketDoesNotExist, result.getVerificationStatus());
145+
assertEquals(0, result.getRequiredSpace());
146+
assertEquals(0, result.getAvailableSpace());
147+
assertFalse(result.containsSufficientSpace());
148+
assertNull(result.getIoException());
149+
}
150+
151+
@Test
152+
public void testObjectsFitBucketWithPathNotDirectory() throws IOException {
153+
final int maxNumBlockAllocationRetries = 1;
154+
final int maxNumObjectTransferAttempts = 1;
155+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
156+
maxNumBlockAllocationRetries,
157+
maxNumObjectTransferAttempts);
158+
159+
final Path textFile = Files.createFile(Paths.get("Gracie.txt"));
160+
161+
try {
162+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
163+
"bad bucket name", Arrays.asList(new String[]{}), textFile);
164+
165+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathIsNotADirectory, result.getVerificationStatus());
166+
assertEquals(0, result.getRequiredSpace());
167+
assertEquals(0, result.getAvailableSpace());
168+
assertFalse(result.containsSufficientSpace());
169+
assertNull(result.getIoException());
170+
} finally {
171+
Files.delete(textFile);
172+
}
173+
}
174+
175+
@Test
176+
public void testObjectsFitBucketPathDoesNotExist() throws IOException {
177+
final int maxNumBlockAllocationRetries = 1;
178+
final int maxNumObjectTransferAttempts = 1;
179+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
180+
maxNumBlockAllocationRetries,
181+
maxNumObjectTransferAttempts);
182+
183+
final Path directory = Files.createDirectory(Paths.get("dir"));
184+
FileUtils.deleteDirectory(directory.toFile());
185+
186+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
187+
"bad bucket name", Arrays.asList(new String[]{}), directory);
188+
189+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathDoesNotExist, result.getVerificationStatus());
190+
assertEquals(0, result.getRequiredSpace());
191+
assertEquals(0, result.getAvailableSpace());
192+
assertFalse(result.containsSufficientSpace());
193+
assertNull(result.getIoException());
194+
}
195+
196+
@Test
197+
public void testObjectsFitBucketPathLacksAccess() throws IOException {
198+
final int maxNumBlockAllocationRetries = 1;
199+
final int maxNumObjectTransferAttempts = 1;
200+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
201+
maxNumBlockAllocationRetries,
202+
maxNumObjectTransferAttempts);
203+
204+
final Path directory = Files.createDirectory(Paths.get("dir"));
205+
directory.toFile().setWritable(false);
206+
207+
try {
208+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
209+
"bad bucket name", Arrays.asList(new String[]{}), directory);
210+
211+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksAccess, result.getVerificationStatus());
212+
assertEquals(0, result.getRequiredSpace());
213+
assertEquals(0, result.getAvailableSpace());
214+
assertFalse(result.containsSufficientSpace());
215+
assertNull(result.getIoException());
216+
} finally {
217+
directory.toFile().setWritable(true);
218+
FileUtils.deleteDirectory(directory.toFile());
219+
}
220+
}
221+
222+
@Test
223+
public void testObjectsFitBucketPathLacksSpace() throws IOException, URISyntaxException {
224+
putObjectThenRunVerification(new MockedFileSystemHelper(), new ResultVerifier() {
225+
@Override
226+
public void verifyResult(final ObjectStorageSpaceVerificationResult result,
227+
final long totalRequiredSize)
228+
{
229+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksSufficientStorageSpace, result.getVerificationStatus());
230+
assertEquals(totalRequiredSize, result.getRequiredSpace());
231+
assertEquals(-1, result.getAvailableSpace());
232+
assertFalse(result.containsSufficientSpace());
233+
assertNull(result.getIoException());
234+
}
235+
});
236+
}
237+
238+
private interface ResultVerifier {
239+
void verifyResult(final ObjectStorageSpaceVerificationResult result, final long totalRequiredSize);
240+
}
241+
242+
private void putObjectThenRunVerification(final FileSystemHelper fileSystemHelper,
243+
final ResultVerifier resultVerifier)
244+
throws IOException, URISyntaxException
245+
{
246+
try {
247+
final String DIR_NAME = "largeFiles/";
248+
final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"};
249+
250+
final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME);
251+
252+
final AtomicLong totalBookSizes = new AtomicLong(0);
253+
254+
final List<String> bookTitles = new ArrayList<>();
255+
final List<Ds3Object> objects = new ArrayList<>();
256+
for (final String book : FILE_NAMES) {
257+
final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book);
258+
final long bookSize = Files.size(objPath);
259+
totalBookSizes.getAndAdd(bookSize);
260+
final Ds3Object obj = new Ds3Object(book, bookSize);
261+
262+
bookTitles.add(book);
263+
objects.add(obj);
264+
}
265+
266+
final int maxNumBlockAllocationRetries = 1;
267+
final int maxNumObjectTransferAttempts = 1;
268+
final int retryDelay = -1;
269+
final Ds3ClientHelpers ds3ClientHelpers = new Ds3ClientHelpersImpl(client,
270+
maxNumBlockAllocationRetries,
271+
maxNumObjectTransferAttempts,
272+
retryDelay,
273+
new SameThreadEventRunner(),
274+
fileSystemHelper);
275+
276+
final AtomicInteger numTimesCallbackCalled = new AtomicInteger(0);
277+
278+
final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
279+
writeJob.attachObjectCompletedListener(new ObjectCompletedListener() {
280+
@Override
281+
public void objectCompleted(final String name) {
282+
numTimesCallbackCalled.getAndIncrement();
283+
284+
final ObjectStorageSpaceVerificationResult result =
285+
ds3ClientHelpers.objectsFromBucketWillFitInDirectory(BUCKET_NAME,
286+
Arrays.asList(FILE_NAMES),
287+
Paths.get("."));
288+
289+
resultVerifier.verifyResult(result, totalBookSizes.get());
290+
}
291+
});
292+
293+
writeJob.transfer(new FileObjectPutter(dirPath));
294+
295+
assertEquals(1, numTimesCallbackCalled.get());
296+
} finally {
297+
deleteAllContents(client, BUCKET_NAME);
298+
}
299+
}
300+
301+
private static class MockedFileSystemHelper extends FileSystemHelperImpl {
302+
@Override
303+
public long getAvailableFileSpace(final Path path) throws IOException {
304+
return -1L;
305+
}
306+
}
307+
308+
@Test
309+
public void testObjectsFitBucketPathThrows() throws IOException, URISyntaxException {
310+
putObjectThenRunVerification(new MockedFileSystemHelperThrows(),
311+
new ResultVerifier() {
312+
@Override
313+
public void verifyResult(final ObjectStorageSpaceVerificationResult result,
314+
final long totalRequiredSize)
315+
{
316+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.CaughtIOException, result.getVerificationStatus());
317+
assertEquals(totalRequiredSize, result.getRequiredSpace());
318+
assertEquals(0, result.getAvailableSpace());
319+
assertFalse(result.containsSufficientSpace());
320+
assertNotNull(result.getIoException());
321+
}
322+
});
323+
}
324+
325+
private static class MockedFileSystemHelperThrows extends FileSystemHelperImpl {
326+
@Override
327+
public long getAvailableFileSpace(final Path path) throws IOException {
328+
throw new IOException("IOExceptionAtor");
329+
}
330+
}
331+
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@
4040
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
4141
import com.spectralogic.ds3client.utils.ResourceUtils;
4242

43-
import com.spectralogic.ds3client.IntValue;
44-
45-
import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimWithFailedChunkAllocation;
4643
import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory.ClientFailureType;
4744

4845
import org.apache.commons.io.FileUtils;

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.io.IOException;
2929
import java.nio.channels.SeekableByteChannel;
3030
import java.nio.file.Path;
31+
import java.util.Collection;
3132
import java.util.Map;
3233
import java.util.UUID;
3334

@@ -382,4 +383,19 @@ public static String stripLeadingPath(final String objectName, final String pref
382383
}
383384
return returnString;
384385
}
386+
387+
/**
388+
* Determine if the file system directory specified in the destinationDirectory parameter
389+
* has enough storage space to contain the objects listed in the parameter objectNames contained in
390+
* the bucket specified in the parameter buckName. You can use this method prior to starting a read
391+
* job to ensure that your file system has enough storage space to contain the objects you wish to
392+
* retrieve.
393+
* @param bucketName The Black Pearl bucket containing the objects you wish to retrieve.
394+
* @param objectNames The names of the objects you wish to retrieve.
395+
* @param destinationDirectory The file system directory in you intend to store retrieved objects.
396+
* @return {@link ObjectStorageSpaceVerificationResult}
397+
*/
398+
public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final String bucketName,
399+
final Collection<String> objectNames,
400+
final Path destinationDirectory);
385401
}

0 commit comments

Comments
 (0)