Skip to content

Commit fbdc527

Browse files
RachelTuckerRachelTucker
committed
Merge branch 'master' into JSDK_192
2 parents aa8d6a7 + 696092d commit fbdc527

28 files changed

+2274
-3
lines changed

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* ****************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.integration;
17+
18+
import com.google.common.collect.ImmutableList;
19+
import com.spectralogic.ds3client.Ds3Client;
20+
import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
21+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
22+
import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
23+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
24+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
25+
import com.spectralogic.ds3client.models.ChecksumType;
26+
import com.spectralogic.ds3client.models.DetailedS3Object;
27+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
28+
import com.spectralogic.ds3client.utils.collections.LazyIterable;
29+
import org.junit.AfterClass;
30+
import org.junit.BeforeClass;
31+
import org.junit.Test;
32+
import org.slf4j.Logger;
33+
import org.slf4j.LoggerFactory;
34+
35+
import java.io.IOException;
36+
import java.util.HashSet;
37+
import java.util.List;
38+
import java.util.Set;
39+
import java.util.UUID;
40+
41+
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
42+
import static com.spectralogic.ds3client.integration.Util.insecureFromEnv;
43+
import static junit.framework.TestCase.assertFalse;
44+
import static org.junit.Assert.assertTrue;
45+
46+
public class SpectraS3PaginationLoader_Test {
47+
48+
private static final Logger LOG = LoggerFactory.getLogger(SpectraS3PaginationLoader_Test.class);
49+
50+
private static final Ds3Client CLIENT = Util.fromEnv();
51+
private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(CLIENT);
52+
private static final String TEST_ENV_NAME = "s3_pagination_test";
53+
private static final int RETRIES = 5;
54+
55+
private static TempStorageIds envStorageIds;
56+
private static UUID envDataPolicyId;
57+
58+
@BeforeClass
59+
public static void startup() throws IOException {
60+
LOG.info("Starting test Setup...");
61+
envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, CLIENT);
62+
envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, CLIENT);
63+
LOG.info("Finished test Setup...");
64+
}
65+
66+
@AfterClass
67+
public static void teardown() throws IOException {
68+
LOG.info("Starting test teardown...");
69+
TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, CLIENT);
70+
CLIENT.close();
71+
LOG.info("Finished test teardown...");
72+
}
73+
74+
@Test
75+
public void basicPagination() throws IOException {
76+
try {
77+
// test setup
78+
HELPERS.ensureBucketExists(TEST_ENV_NAME);
79+
final int numObjects = 55;
80+
CLIENT.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(TEST_ENV_NAME, createTestList(numObjects)));
81+
82+
final GetObjectsFullDetailsLoaderFactory loaderFactory = new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, "", 10, RETRIES, false);
83+
84+
final Set<String> foundItems = new HashSet<>();
85+
86+
for (final DetailedS3Object detailedS3Object : new LazyIterable<>(loaderFactory)) {
87+
LOG.info("Testing: {}", detailedS3Object.getName());
88+
assertFalse(foundItems.contains(detailedS3Object.getName()));
89+
foundItems.add(detailedS3Object.getName());
90+
}
91+
92+
for (int i = 0; i < numObjects; i++) {
93+
assertTrue(foundItems.contains("obj." + i));
94+
}
95+
96+
} finally {
97+
deleteAllContents(CLIENT, TEST_ENV_NAME);
98+
}
99+
}
100+
101+
private List<Ds3Object> createTestList(final int numObjects) {
102+
final ImmutableList.Builder<Ds3Object> builder = ImmutableList.builder();
103+
for (int i = 0; i < numObjects; i++) {
104+
builder.add(new Ds3Object("obj." + i, i+1));
105+
}
106+
107+
return builder.build();
108+
}
109+
}

‎ds3-sdk/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,5 +80,7 @@ dependencies {
8080
compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.7.1'
8181
compile 'com.google.guava:guava:19.0'
8282
compile 'com.google.code.findbugs:annotations:3.0.1'
83+
compile group: 'net.java.dev.jna', name: 'jna-platform', version: '4.2.2'
84+
compile group: 'net.java.dev.jna', name: 'jna', version: '4.2.2'
8385
testCompile 'org.hamcrest:hamcrest-library:1.3'
8486
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoaderFactory.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
import com.spectralogic.ds3client.models.DetailedS3Object;
2020
import com.spectralogic.ds3client.utils.collections.LazyIterable;
2121

22+
/**
23+
* A lazy implementation for paginating the Spectra S3 {@link com.spectralogic.ds3client.commands.spectrads3.GetObjectsWithFullDetailsSpectraS3Request} api call
24+
*/
2225
public class GetObjectsFullDetailsLoaderFactory implements LazyIterable.LazyLoaderFactory<DetailedS3Object> {
2326
private final Ds3Client client;
2427
private final String bucket;

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/SpectraS3PaginationLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public List<T> getNextValues() {
6161
final E request = paginatingCommand.createRequest();
6262

6363
request.withPageLength(pageLength)
64-
.withPageOffset(pageOffset);
64+
.withPageOffset(pageOffset * pageLength);
6565

6666
try {
6767
final F response = paginatingCommand.invokeCommand(request);

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/metadata/MACMetadataRestore.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.metadata;
17+
18+
import com.spectralogic.ds3client.metadata.interfaces.MetadataRestoreListener;
19+
import com.spectralogic.ds3client.networking.Metadata;
20+
21+
import java.text.SimpleDateFormat;
22+
import java.util.Calendar;
23+
24+
import static com.spectralogic.ds3client.utils.MetadataKeyConstants.*;
25+
26+
27+
public class MACMetadataRestore extends PosixMetadataRestore {
28+
public MACMetadataRestore(final Metadata metadata, final String filePath, final String localOS, final MetadataRestoreListener metadataRestoreListener) {
29+
super(metadata, filePath, localOS, metadataRestoreListener);
30+
}
31+
32+
/**
33+
* Return date in specified format.
34+
*
35+
* @param milliSeconds Date in milliseconds
36+
* @param dateFormat Date format
37+
* @return String representing date in specified format
38+
*/
39+
public static String getDate(final long milliSeconds, final String dateFormat) {
40+
// Create a DateFormatter object for displaying date in specified format.
41+
final SimpleDateFormat formatter = new SimpleDateFormat(dateFormat);
42+
// Create a calendar object that will convert the date and time value in milliseconds to date.
43+
final Calendar calendar = Calendar.getInstance();
44+
calendar.setTimeInMillis(milliSeconds);
45+
return formatter.format(calendar.getTime());
46+
}
47+
48+
@Override
49+
public void restoreFileTimes() {
50+
String creationTime = null;
51+
if (metadata.get(KEY_CREATION_TIME).size() > 0) {
52+
creationTime = metadata.get(KEY_CREATION_TIME).get(0);
53+
}
54+
String accessTime = null;
55+
if (metadata.get(KEY_ACCESS_TIME).size() > 0) {
56+
accessTime = metadata.get(KEY_ACCESS_TIME).get(0);
57+
}
58+
String modifiedTime = null;
59+
if (metadata.get(KEY_LAST_MODIFIED_TIME).size() > 0) {
60+
modifiedTime = metadata.get(KEY_LAST_MODIFIED_TIME).get(0);
61+
}
62+
if (modifiedTime != null && creationTime != null && accessTime != null) {
63+
restoreCreationTimeMAC(objectName, creationTime);
64+
restoreModifiedTimeMAC(objectName, modifiedTime);
65+
}
66+
67+
}
68+
69+
/**
70+
* Restore creation time in mac only if target creation is before current creation time
71+
*
72+
* @param objectName path of the object where we need to restore
73+
* @param creationTime creation time got from server
74+
*/
75+
private void restoreCreationTimeMAC(final String objectName, final String creationTime) {
76+
try {
77+
final ProcessBuilder processBuilder = new ProcessBuilder("touch", "-t", getDate(Long.parseLong(creationTime), "YYYYMMddHHmm"), objectName);
78+
final Process process = processBuilder.start();
79+
//Wait to get exit value
80+
final int exitValue = process.waitFor();
81+
if(exitValue != 0) {
82+
LOG.error("Unable to restore creation time::"+exitValue);
83+
metadataRestoreListener.metadataRestoreFailed("Unable to restore creation time ::"+exitValue);
84+
}
85+
} catch (final Exception e) {
86+
LOG.error("Unable to restore creation time", e);
87+
metadataRestoreListener.metadataRestoreFailed("Unable to restore creation time ::"+e.getMessage());
88+
}
89+
90+
}
91+
92+
/**
93+
* Restore modified time in case of MAC using touch command
94+
*
95+
* @param objectName path of the object where we need to restore
96+
* @param modifiedTime modified time need to restore
97+
*/
98+
private void restoreModifiedTimeMAC(final String objectName, final String modifiedTime) {
99+
try {
100+
final ProcessBuilder processBuilder = new ProcessBuilder("touch", "-mt", getDate(Long.parseLong(modifiedTime), "YYYYMMddHHmm"), objectName);
101+
final Process process = processBuilder.start();
102+
//Wait to get exit value
103+
final int exitValue = process.waitFor();
104+
if(exitValue != 0) {
105+
LOG.error("Unable to restore modified time::"+exitValue);
106+
metadataRestoreListener.metadataRestoreFailed("Unable to restore creation time ::"+exitValue);
107+
}
108+
} catch (final Exception e) {
109+
LOG.error("Unable to restore modified time", e);
110+
metadataRestoreListener.metadataRestoreFailed("Unable to restore modified time ::"+e.getMessage());
111+
}
112+
113+
}
114+
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.metadata;
17+
18+
19+
import com.google.common.collect.ImmutableMap;
20+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
21+
import com.spectralogic.ds3client.metadata.interfaces.MetadataStore;
22+
import com.spectralogic.ds3client.metadata.interfaces.MetadataStoreListener;
23+
import com.spectralogic.ds3client.utils.MetaDataUtil;
24+
import org.slf4j.Logger;
25+
import org.slf4j.LoggerFactory;
26+
27+
import java.io.IOException;
28+
import java.nio.file.Files;
29+
import java.nio.file.Path;
30+
import java.nio.file.attribute.BasicFileAttributes;
31+
import java.nio.file.attribute.PosixFileAttributes;
32+
import java.util.Map;
33+
import java.util.Set;
34+
35+
/**
36+
* Implementation of MetaDataAcess Interface
37+
* Used to store meta data on Server
38+
*/
39+
public class MetadataAccessImpl implements Ds3ClientHelpers.MetadataAccess {
40+
static private final Logger LOG = LoggerFactory.getLogger(MetadataAccessImpl.class);
41+
private final Map<String, Path> fileMapper;
42+
private final MetadataStoreListener metadataStoreListener;
43+
44+
public MetadataAccessImpl(final Map<String, Path> fileMapper , final MetadataStoreListener metadataStoreListener) {
45+
this.fileMapper = fileMapper;
46+
this.metadataStoreListener = metadataStoreListener;
47+
}
48+
49+
@Override
50+
public Map<String, String> getMetadataValue(final String filename) {
51+
try {
52+
final Path file = fileMapper.get(filename);
53+
return storeMetaData(file).build();
54+
} catch (final Exception e) {
55+
LOG.error("failed to store Metadata", e);
56+
metadataStoreListener.onMetadataFailed("Unable to get MetaData"+e.getMessage());
57+
return null;
58+
}
59+
}
60+
61+
/**
62+
* @param file local path of file
63+
* @return map builder containing the data to be stored on server
64+
*/
65+
private ImmutableMap.Builder<String, String> storeMetaData(final Path file) {
66+
final ImmutableMap.Builder<String, String> metadata = new ImmutableMap.Builder<>();
67+
new ImmutableMap.Builder<String, String>();
68+
try {
69+
//get local os name
70+
final String localOSName = MetaDataUtil.getOS();
71+
final Set<String> setFileAttributes = MetaDataUtil.getSupportedFileAttributes(file);
72+
//get metadata store based on os type
73+
final MetadataStore metadataStore = new MetadataStoreFactory().getOsSpecificMetadataStore(localOSName, metadata,metadataStoreListener);
74+
metadataStore.saveOSMetaData(localOSName);
75+
76+
final BasicFileAttributes attr = Files.readAttributes(file, BasicFileAttributes.class);
77+
final PosixFileAttributes attrPosix = Files.readAttributes(file, PosixFileAttributes.class);
78+
79+
metadataStore.saveCreationTimeMetaData(attr);
80+
metadataStore.saveAccessTimeMetaData(attr);
81+
metadataStore.saveLastModifiedTime(attr);
82+
83+
if(attrPosix != null)
84+
metadataStore.saveOSSpecificMetadata(file,attrPosix);
85+
else
86+
metadataStore.saveOSSpecificMetadata(file,attr);
87+
88+
} catch (final IOException ioe) {
89+
LOG.error("unable to get metadata", ioe);
90+
metadataStoreListener.onMetadataFailed("Unable to get MetaData"+ioe.getMessage());
91+
}
92+
return metadata;
93+
}
94+
95+
}

0 commit comments

Comments
 (0)