Skip to content

Commit 024843d

Browse files
GraciesPadreGraciesPadre
authored
Merge pull request #444 from rpmoore/metadata_updates
Metadata updates
2 parents 7195b57 + c7d3fa7 commit 024843d

File tree

13 files changed

+229
-40
lines changed

13 files changed

+229
-40
lines changed

‎ds3-interfaces/build.gradle‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,12 @@ buildscript {
2222

2323
apply plugin: 'com.github.johnrengelman.shadow'
2424

25+
dependencies {
26+
compile 'commons-io:commons-io:2.4'
27+
}
28+
2529
shadowJar {
30+
relocate 'org.apache', 'ds3fatjar.org.apache'
2631
dependencies {
2732
exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
2833
exclude(dependency('org.mockito:mockito-core:1.10.19'))

‎ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* ****************************************************************************
3+
* Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.exceptions;
17+
18+
import org.apache.commons.io.output.StringBuilderWriter;
19+
20+
import java.io.PrintWriter;
21+
import java.util.ArrayList;
22+
import java.util.List;
23+
24+
public class AggregateException extends RuntimeException {
25+
26+
private final List<Throwable> exceptions;
27+
28+
public AggregateException(final Iterable<Throwable> exceptions) {
29+
super("One or more exceptions were aggregated together");
30+
this.exceptions = exceptionList(exceptions);
31+
}
32+
33+
private static List<Throwable> exceptionList(final Iterable<Throwable> exceptions) {
34+
final List<Throwable> exceptionList = new ArrayList<>();
35+
for (final Throwable t : exceptions) {
36+
exceptionList.add(t);
37+
}
38+
return exceptionList;
39+
}
40+
41+
public Iterable<Throwable> getExceptions() {
42+
return exceptions;
43+
}
44+
45+
public String toString() {
46+
final StringBuilder builder = new StringBuilder();
47+
builder.append("One or more exceptions were aggregated:");
48+
49+
try (final StringBuilderWriter writer = new StringBuilderWriter(builder);
50+
final PrintWriter printWriter = new PrintWriter(writer)) {
51+
52+
for (final Throwable t : exceptions) {
53+
printWriter.append("Exception: ").append(t.getMessage());
54+
t.printStackTrace(printWriter);
55+
}
56+
}
57+
58+
return builder.toString();
59+
}
60+
}

‎ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers/MetadataAccess.java‎ renamed to ‎ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/MetadataAccess.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* ****************************************************************************
1414
*/
1515

16-
package com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
16+
package com.spectralogic.ds3client.helpers;
1717

1818
import java.util.Map;
1919

‎ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* ****************************************************************************
3+
* Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.exceptions;
17+
18+
import org.junit.Test;
19+
20+
import java.util.ArrayList;
21+
import java.util.Iterator;
22+
import java.util.List;
23+
24+
import static org.hamcrest.CoreMatchers.is;
25+
import static org.hamcrest.CoreMatchers.notNullValue;
26+
import static org.junit.Assert.assertFalse;
27+
import static org.junit.Assert.assertThat;
28+
import static org.junit.Assert.assertTrue;
29+
30+
public class AggregateException_Test {
31+
32+
@Test
33+
public void basicAggregate() {
34+
final Exception e1 = new Exception("first exception");
35+
final Exception e2 = new Exception("second exception");
36+
37+
final List<Throwable> exceptionList = new ArrayList<>();
38+
exceptionList.add(e1);
39+
exceptionList.add(e2);
40+
41+
final AggregateException aggregateException = new AggregateException(exceptionList);
42+
43+
final Iterator<Throwable> exceptionIter = aggregateException.getExceptions().iterator();
44+
45+
assertTrue(exceptionIter.hasNext());
46+
assertThat(exceptionIter.next(), is(notNullValue()));
47+
assertTrue(exceptionIter.hasNext());
48+
assertThat(exceptionIter.next(), is(notNullValue()));
49+
assertFalse(exceptionIter.hasNext());
50+
}
51+
52+
@Test
53+
public void repeatIteration() {
54+
final Exception e1 = new Exception("first exception");
55+
56+
final List<Throwable> exceptionList = new ArrayList<>();
57+
exceptionList.add(e1);
58+
59+
final AggregateException aggregateException = new AggregateException(exceptionList);
60+
61+
final Iterator<Throwable> exceptionIter = aggregateException.getExceptions().iterator();
62+
63+
assertTrue(exceptionIter.hasNext());
64+
assertThat(exceptionIter.next(), is(notNullValue()));
65+
assertFalse(exceptionIter.hasNext());
66+
67+
final Iterator<Throwable> repeatIter = aggregateException.getExceptions().iterator();
68+
assertTrue(repeatIter.hasNext());
69+
assertThat(repeatIter.next(), is(notNullValue()));
70+
assertFalse(repeatIter.hasNext());
71+
72+
}
73+
}

‎ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818

1919
import com.google.common.collect.ImmutableMap;
20-
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.MetadataAccess;
20+
import com.spectralogic.ds3client.helpers.MetadataAccess;
2121
import com.spectralogic.ds3client.metadata.interfaces.MetadataStore;
2222
import com.spectralogic.ds3client.utils.Platform;
2323

‎ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataReceivedListenerImpl.java‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,24 @@
1515

1616
package com.spectralogic.ds3client.metadata;
1717

18+
import com.google.common.collect.ImmutableList;
19+
import com.spectralogic.ds3client.exceptions.AggregateException;
1820
import com.spectralogic.ds3client.helpers.MetadataReceivedListener;
1921
import com.spectralogic.ds3client.metadata.interfaces.MetadataRestore;
2022
import com.spectralogic.ds3client.networking.Metadata;
23+
import com.spectralogic.ds3client.utils.Guard;
24+
import org.slf4j.Logger;
25+
import org.slf4j.LoggerFactory;
2126

2227
import java.io.IOException;
2328

2429
public class MetadataReceivedListenerImpl implements MetadataReceivedListener {
30+
31+
private final static Logger LOG = LoggerFactory.getLogger(MetadataReceivedListenerImpl.class);
32+
2533
private final String localFilePath;
2634

35+
2736
public MetadataReceivedListenerImpl(final String localFilePath) {
2837
this.localFilePath = localFilePath;
2938
}
@@ -45,17 +54,41 @@ public void metadataReceived(final String filename, final Metadata metadata) {
4554
* @param metadata metadata which needs to be set on local file
4655
*/
4756
private void restoreMetaData(final String objectName, final Metadata metadata) throws IOException, InterruptedException {
57+
58+
final ImmutableList.Builder<Throwable> exceptionBuilder = ImmutableList.builder();
59+
4860
//get metadatarestore on the basis of os
4961
final MetadataRestore metadataRestore = new MetadataRestoreFactory().getOSSpecificMetadataRestore(metadata, objectName);
5062
//restore os name
5163
metadataRestore.restoreOSName();
64+
5265
//restore user and owner based on OS
53-
metadataRestore.restoreUserAndOwner();
66+
try {
67+
metadataRestore.restoreUserAndOwner();
68+
} catch (final Throwable t) {
69+
LOG.error("Could not restore owner and owner information", t);
70+
exceptionBuilder.add(t);
71+
}
72+
5473
//restore creation and modified time based on OS
55-
metadataRestore.restoreFileTimes();
56-
//restore permissions based on OS
57-
metadataRestore.restorePermissions();
58-
}
74+
try {
75+
metadataRestore.restoreFileTimes();
76+
} catch (final Throwable t) {
77+
LOG.error("Could not restore the file times", t);
78+
exceptionBuilder.add(t);
79+
}
5980

81+
//restore permissions based on OS
82+
try {
83+
metadataRestore.restorePermissions();
84+
} catch (final Throwable t) {
85+
LOG.error("Could not restore the file permissions", t);
86+
exceptionBuilder.add(t);
87+
}
6088

89+
final ImmutableList<Throwable> exceptions = exceptionBuilder.build();
90+
if (!Guard.isNullOrEmpty(exceptions)) {
91+
throw new AggregateException(exceptions);
92+
}
93+
}
6194
}

‎ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore.java‎

Lines changed: 44 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@
1818
import com.google.common.collect.ImmutableList;
1919
import com.spectralogic.ds3client.metadata.jna.Advapi32;
2020
import com.spectralogic.ds3client.networking.Metadata;
21+
import com.spectralogic.ds3client.utils.Guard;
2122
import com.sun.jna.platform.win32.WinNT;
23+
import org.slf4j.Logger;
24+
import org.slf4j.LoggerFactory;
25+
2226
import java.io.File;
2327
import java.io.IOException;
2428
import java.nio.file.FileSystems;
@@ -36,6 +40,9 @@
3640
import static com.spectralogic.ds3client.metadata.MetadataKeyConstants.KEY_OWNER;
3741

3842
class WindowsMetadataRestore extends AbstractMetadataRestore {
43+
44+
private static final Logger LOG = LoggerFactory.getLogger(WindowsMetadataRestore.class);
45+
3946
WindowsMetadataRestore(final Metadata metadata, final String filePath, final String localOS) {
4047
this.metadata = metadata;
4148
this.objectName = filePath;
@@ -44,26 +51,31 @@ class WindowsMetadataRestore extends AbstractMetadataRestore {
4451

4552
@Override
4653
public void restoreUserAndOwner() throws IOException {
47-
if (storedOS.equals(localOS)) {
48-
String ownerSid = null;
49-
if (metadata.get(KEY_OWNER).size() > 0) {
50-
ownerSid = metadata.get(KEY_OWNER).get(0);
51-
}
52-
String groupSid = null;
53-
if (metadata.get(KEY_GROUP).size() > 0) {
54-
groupSid = metadata.get(KEY_GROUP).get(0);
55-
}
56-
if (ownerSid != null && groupSid != null && !ownerSid.equals("") && !groupSid.equals("")) {
57-
setOwnerIdandGroupId(ownerSid, groupSid);
54+
if (storedOS != null && storedOS.equals(localOS)) {
55+
56+
final String ownerSid = getMetadataProperty(metadata, KEY_OWNER);
57+
final String groupSid = getMetadataProperty(metadata, KEY_GROUP);
58+
59+
if (!Guard.isStringNullOrEmpty(ownerSid) && !Guard.isStringNullOrEmpty(groupSid)) {
60+
setOwnerIdAndGroupId(ownerSid, groupSid);
61+
} else {
62+
LOG.warn("Cannot determine owner or group settings for {}", this.objectName);
5863
}
64+
} else {
65+
LOG.warn("The OS settings for owner and group properties cannot be restored for {}", this.objectName);
5966
}
67+
}
6068

69+
private static String getMetadataProperty(final Metadata metadata, final String metadataName) {
70+
return metadata.get(metadataName).get(0);
6171
}
6272

6373
@Override
6474
public void restorePermissions() throws IOException, InterruptedException {
6575
if (storedOS != null && storedOS.equals(localOS)) {
6676
setPermissionsForWindows();
77+
} else {
78+
LOG.warn("The OS settings for the file permissions cannot be restored for {}", this.objectName);
6779
}
6880
restoreFlags();
6981
}
@@ -76,19 +88,26 @@ private void setPermissionsForWindows() throws IOException {
7688
final String userListDisplay;
7789
final String[] users;
7890
final String[] usersDisplay;
79-
if (metadata.get("ds3-userList").size() > 0) {
91+
if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-userList"))) {
8092
userList = metadata.get("ds3-userList").get(0);
81-
if (metadata.get("ds3-userListDisplay").size() > 0) {
93+
if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-userListDisplay"))) {
8294
userListDisplay = metadata.get("ds3-userListDisplay").get(0);
8395
users = userList.split("-");
8496
usersDisplay = userListDisplay.split("-");
8597
for (int i = 0; i < users.length; i++) {
86-
if (metadata.get("ds3-" + users[i]).size() > 0) {
87-
final String ownerPermission = metadata.get("ds3-" + users[i]).get(0);
98+
final String user = users[i];
99+
if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-" + user))) {
100+
final String ownerPermission = metadata.get("ds3-" + user).get(0);
88101
restorePermissionByUser(ownerPermission, usersDisplay[i], aclEntryBuilder);
102+
} else {
103+
LOG.warn("Did not find any permissions for {} for file {}", user, this.objectName);
89104
}
90105
}
106+
} else {
107+
LOG.warn("There was not a 'userListDisplay' metadata entry for file {}, so we will not restore any permissions", this.objectName);
91108
}
109+
} else {
110+
LOG.warn("There was not a 'userList' metadata entry for file {}, so we will not restore any permissions", this.objectName);
92111
}
93112

94113
aclAttributeView.setAcl(aclEntryBuilder.build());
@@ -148,7 +167,7 @@ private void restorePermissionByUser(final String permission,
148167
* @param ownerSidId sid of the owner
149168
* @param groupSidId sid of the group
150169
*/
151-
private void setOwnerIdandGroupId(final String ownerSidId, final String groupSidId) throws IOException {
170+
private void setOwnerIdAndGroupId(final String ownerSidId, final String groupSidId) throws IOException {
152171
final int infoType = WinNT.OWNER_SECURITY_INFORMATION | WinNT.GROUP_SECURITY_INFORMATION;
153172
final WinNT.PSIDByReference referenceOwner = new WinNT.PSIDByReference();
154173
Advapi32.INSTANCE.ConvertStringSidToSid(ownerSidId, referenceOwner);
@@ -166,7 +185,11 @@ private void restoreFlags() throws IOException, InterruptedException {
166185
if (metadata.get(KEY_FLAGS).size() > 0) {
167186
final String flags = metadata.get(KEY_FLAGS).get(0);
168187
restoreFlagsWindows(flags);
188+
} else {
189+
LOG.warn("The file flag settings do not exist for file {} and will not be restored", this.objectName);
169190
}
191+
} else {
192+
LOG.warn("The OS settings for restoring the file flags cannot be done for {}", this.objectName);
170193
}
171194
}
172195

@@ -199,9 +222,12 @@ private void restoreFlagsWindows(final String flag) throws IOException, Interrup
199222
stringBuilder.append(" -I");
200223
stringBuilder.append(" -H");
201224
}
202-
stringBuilder.append(" " + "\"" + objectName + "\"");
225+
stringBuilder.append(" " + "\"").append(objectName).append("\"");
203226

204227
final Process p = Runtime.getRuntime().exec(stringBuilder.toString().split(" "));
205-
p.waitFor();
228+
final int returnCode = p.waitFor();
229+
if (returnCode != 0) {
230+
LOG.error("Restoring the flag settings for file {} was not successful", this.objectName);
231+
}
206232
}
207233
}

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,7 @@
2222
import com.spectralogic.ds3client.commands.*;
2323
import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
2424
import com.spectralogic.ds3client.commands.spectrads3.*;
25-
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
26-
import com.spectralogic.ds3client.helpers.JobRecoveryException;
27-
import com.spectralogic.ds3client.helpers.JobRecoveryNotActiveException;
28-
import com.spectralogic.ds3client.helpers.ObjectCompletedListener;
25+
import com.spectralogic.ds3client.helpers.*;
2926
import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
3027
import com.spectralogic.ds3client.integration.test.helpers.JobStatusHelper;
3128
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -1178,7 +1175,7 @@ public void testHelperMetadata() throws IOException, URISyntaxException {
11781175

11791176
final AtomicBoolean calledWithMetadata = new AtomicBoolean(false);
11801177

1181-
job.withMetadata(new Ds3ClientHelpers.MetadataAccess() {
1178+
job.withMetadata(new MetadataAccess() {
11821179
@Override
11831180
public Map<String, String> getMetadataValue(final String filename) {
11841181
if (filename.equals("beowulf.txt")) {

0 commit comments

Comments
 (0)