Skip to content

Commit 3f53d8c

Browse files
sulabhjain1991sulabhjain1991
committed
Merge remote-tracking branch 'origin/master'
2 parents 1072beb + 67747f1 commit 3f53d8c

35 files changed

+22709
-74
lines changed

‎docker/Dockerfile‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
1-
FROM greyrocksoftware/javastandalone
1+
FROM greyrocksoftware/javastandalone7
22

3-
RUN apt-get update
3+
# Set the locale
4+
RUN locale-gen en_US.UTF-8
5+
ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
6+
7+
RUN apt-get update && apt-get install -y \
8+
git
49

5-
RUN apt-get install git -y
610

711
ADD run_tests.sh /opt/
812

‎ds3-sdk-integration/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ dependencies {
1717
compile 'commons-codec:commons-codec:1.10'
1818
compile 'junit:junit:4.12'
1919
testCompile 'org.hamcrest:hamcrest-library:1.3'
20+
testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0'
2021
}
2122

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
/*
2+
* ****************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.helpers;
17+
18+
import com.spectralogic.ds3client.Ds3Client;
19+
import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
20+
import com.spectralogic.ds3client.integration.Util;
21+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
22+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
23+
import com.spectralogic.ds3client.models.ChecksumType;
24+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
25+
import com.spectralogic.ds3client.utils.ResourceUtils;
26+
import org.apache.commons.io.FileUtils;
27+
import org.junit.AfterClass;
28+
import org.junit.Before;
29+
import org.junit.BeforeClass;
30+
import org.junit.Test;
31+
32+
import java.io.IOException;
33+
import java.net.URISyntaxException;
34+
import java.nio.file.Files;
35+
import java.nio.file.Path;
36+
import java.nio.file.Paths;
37+
import java.util.*;
38+
import java.util.concurrent.atomic.AtomicInteger;
39+
import java.util.concurrent.atomic.AtomicLong;
40+
41+
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
42+
43+
import static org.junit.Assert.assertEquals;
44+
import static org.junit.Assert.assertTrue;
45+
import static org.junit.Assert.assertFalse;
46+
import static org.junit.Assert.assertNull;
47+
import static org.junit.Assert.assertNotNull;
48+
49+
public class FileSystemHelper_Test {
50+
private static final Ds3Client client = Util.fromEnv();
51+
private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client);
52+
private static final String BUCKET_NAME = "File_System_Helper_Test";
53+
private static final String TEST_ENV_NAME = "FileSystem_Helper_Test";
54+
private static TempStorageIds envStorageIds;
55+
private static UUID envDataPolicyId;
56+
57+
@BeforeClass
58+
public static void startup() throws IOException {
59+
envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client);
60+
envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, client);
61+
}
62+
63+
@Before
64+
public void setupBucket() throws IOException {
65+
HELPERS.ensureBucketExists(BUCKET_NAME, envDataPolicyId);
66+
}
67+
68+
@AfterClass
69+
public static void teardown() throws IOException {
70+
TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, client);
71+
client.close();
72+
}
73+
74+
@Test
75+
public void testObjectsFitBucketThatHasContent() throws IOException, URISyntaxException {
76+
putObjectThenRunVerification(new FileSystemHelperImpl(), new ResultVerifier() {
77+
@Override
78+
public void verifyResult(final ObjectStorageSpaceVerificationResult result, final long totalRequiredSize) {
79+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.OK, result.getVerificationStatus());
80+
assertEquals(result.getRequiredSpace(), totalRequiredSize);
81+
assertTrue(result.getAvailableSpace() > 0);
82+
assertTrue(result.containsSufficientSpace());
83+
assertNull(result.getIoException());
84+
}
85+
});
86+
}
87+
88+
@Test
89+
public void testObjectsFitBucketWithNonExistentBucket() {
90+
final int maxNumBlockAllocationRetries = 1;
91+
final int maxNumObjectTransferAttempts = 1;
92+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
93+
maxNumBlockAllocationRetries,
94+
maxNumObjectTransferAttempts);
95+
96+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
97+
"bad bucket name", Arrays.asList(new String[] {}), Paths.get("."));
98+
99+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.BucketDoesNotExist, result.getVerificationStatus());
100+
assertEquals(0, result.getRequiredSpace());
101+
assertEquals(0, result.getAvailableSpace());
102+
assertFalse(result.containsSufficientSpace());
103+
assertNull(result.getIoException());
104+
}
105+
106+
@Test
107+
public void testObjectsFitBucketWithPathNotDirectory() throws IOException {
108+
final int maxNumBlockAllocationRetries = 1;
109+
final int maxNumObjectTransferAttempts = 1;
110+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
111+
maxNumBlockAllocationRetries,
112+
maxNumObjectTransferAttempts);
113+
114+
final Path textFile = Files.createFile(Paths.get("Gracie.txt"));
115+
116+
try {
117+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
118+
"bad bucket name", Arrays.asList(new String[]{}), textFile);
119+
120+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathIsNotADirectory, result.getVerificationStatus());
121+
assertEquals(0, result.getRequiredSpace());
122+
assertEquals(0, result.getAvailableSpace());
123+
assertFalse(result.containsSufficientSpace());
124+
assertNull(result.getIoException());
125+
} finally {
126+
Files.delete(textFile);
127+
}
128+
}
129+
130+
@Test
131+
public void testObjectsFitBucketPathDoesNotExist() throws IOException {
132+
final int maxNumBlockAllocationRetries = 1;
133+
final int maxNumObjectTransferAttempts = 1;
134+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
135+
maxNumBlockAllocationRetries,
136+
maxNumObjectTransferAttempts);
137+
138+
final Path directory = Files.createDirectory(Paths.get("dir"));
139+
FileUtils.deleteDirectory(directory.toFile());
140+
141+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
142+
"bad bucket name", Arrays.asList(new String[]{}), directory);
143+
144+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathDoesNotExist, result.getVerificationStatus());
145+
assertEquals(0, result.getRequiredSpace());
146+
assertEquals(0, result.getAvailableSpace());
147+
assertFalse(result.containsSufficientSpace());
148+
assertNull(result.getIoException());
149+
}
150+
151+
@Test
152+
public void testObjectsFitBucketPathLacksAccess() throws IOException, InterruptedException {
153+
final int maxNumBlockAllocationRetries = 1;
154+
final int maxNumObjectTransferAttempts = 1;
155+
final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
156+
maxNumBlockAllocationRetries,
157+
maxNumObjectTransferAttempts);
158+
159+
final Path directory = Files.createDirectory(Paths.get("dir"));
160+
if (org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS) {
161+
// Deny write data access to everyone, making the directory unwritable
162+
Runtime.getRuntime().exec("icacls dir /deny Everyone:(WD)").waitFor();
163+
} else {
164+
directory.toFile().setWritable(false);
165+
}
166+
167+
try {
168+
final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory(
169+
"bad bucket name", Arrays.asList(new String[]{}), directory);
170+
171+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksAccess, result.getVerificationStatus());
172+
assertEquals(0, result.getRequiredSpace());
173+
assertEquals(0, result.getAvailableSpace());
174+
assertFalse(result.containsSufficientSpace());
175+
assertNull(result.getIoException());
176+
} finally {
177+
if (org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS) {
178+
// Grant write data access to everyone, making the directory writable, so we can delete it.
179+
Runtime.getRuntime().exec("icacls dir /grant Everyone:(WD)").waitFor();
180+
} else {
181+
directory.toFile().setWritable(true);
182+
}
183+
184+
FileUtils.deleteDirectory(directory.toFile());
185+
}
186+
}
187+
188+
@Test
189+
public void testObjectsFitBucketPathLacksSpace() throws IOException, URISyntaxException {
190+
putObjectThenRunVerification(new MockedFileSystemHelper(), new ResultVerifier() {
191+
@Override
192+
public void verifyResult(final ObjectStorageSpaceVerificationResult result,
193+
final long totalRequiredSize)
194+
{
195+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksSufficientStorageSpace, result.getVerificationStatus());
196+
assertEquals(totalRequiredSize, result.getRequiredSpace());
197+
assertEquals(-1, result.getAvailableSpace());
198+
assertFalse(result.containsSufficientSpace());
199+
assertNull(result.getIoException());
200+
}
201+
});
202+
}
203+
204+
private interface ResultVerifier {
205+
void verifyResult(final ObjectStorageSpaceVerificationResult result, final long totalRequiredSize);
206+
}
207+
208+
private void putObjectThenRunVerification(final FileSystemHelper fileSystemHelper,
209+
final ResultVerifier resultVerifier)
210+
throws IOException, URISyntaxException
211+
{
212+
try {
213+
final String DIR_NAME = "largeFiles/";
214+
final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"};
215+
216+
final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME);
217+
218+
final AtomicLong totalBookSizes = new AtomicLong(0);
219+
220+
final List<String> bookTitles = new ArrayList<>();
221+
final List<Ds3Object> objects = new ArrayList<>();
222+
for (final String book : FILE_NAMES) {
223+
final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book);
224+
final long bookSize = Files.size(objPath);
225+
totalBookSizes.getAndAdd(bookSize);
226+
final Ds3Object obj = new Ds3Object(book, bookSize);
227+
228+
bookTitles.add(book);
229+
objects.add(obj);
230+
}
231+
232+
final int maxNumBlockAllocationRetries = 1;
233+
final int maxNumObjectTransferAttempts = 1;
234+
final int retryDelay = -1;
235+
final Ds3ClientHelpers ds3ClientHelpers = new Ds3ClientHelpersImpl(client,
236+
maxNumBlockAllocationRetries,
237+
maxNumObjectTransferAttempts,
238+
retryDelay,
239+
new SameThreadEventRunner(),
240+
fileSystemHelper);
241+
242+
final AtomicInteger numTimesCallbackCalled = new AtomicInteger(0);
243+
244+
final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
245+
writeJob.attachObjectCompletedListener(new ObjectCompletedListener() {
246+
@Override
247+
public void objectCompleted(final String name) {
248+
numTimesCallbackCalled.getAndIncrement();
249+
250+
final ObjectStorageSpaceVerificationResult result =
251+
ds3ClientHelpers.objectsFromBucketWillFitInDirectory(BUCKET_NAME,
252+
Arrays.asList(FILE_NAMES),
253+
Paths.get("."));
254+
255+
resultVerifier.verifyResult(result, totalBookSizes.get());
256+
}
257+
});
258+
259+
writeJob.transfer(new FileObjectPutter(dirPath));
260+
261+
assertEquals(1, numTimesCallbackCalled.get());
262+
} finally {
263+
deleteAllContents(client, BUCKET_NAME);
264+
}
265+
}
266+
267+
private static class MockedFileSystemHelper extends FileSystemHelperImpl {
268+
@Override
269+
public long getAvailableFileSpace(final Path path) throws IOException {
270+
return -1L;
271+
}
272+
}
273+
274+
@Test
275+
public void testObjectsFitBucketPathThrows() throws IOException, URISyntaxException {
276+
putObjectThenRunVerification(new MockedFileSystemHelperThrows(),
277+
new ResultVerifier() {
278+
@Override
279+
public void verifyResult(final ObjectStorageSpaceVerificationResult result,
280+
final long totalRequiredSize)
281+
{
282+
assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.CaughtIOException, result.getVerificationStatus());
283+
assertEquals(totalRequiredSize, result.getRequiredSpace());
284+
assertEquals(0, result.getAvailableSpace());
285+
assertFalse(result.containsSufficientSpace());
286+
assertNotNull(result.getIoException());
287+
}
288+
});
289+
}
290+
291+
private static class MockedFileSystemHelperThrows extends FileSystemHelperImpl {
292+
@Override
293+
public long getAvailableFileSpace(final Path path) throws IOException {
294+
throw new IOException("IOExceptionAtor");
295+
}
296+
}
297+
}

0 commit comments

Comments
 (0)