Skip to content

Commit 17e2616

Browse files
rpmoorerpmoore
authored
Merge pull request #324 from berkeleysquare/master
JAVACLI86 -- escape ? % and ; in object names
2 parents 3e3c019 + 166d891 commit 17e2616

File tree

6 files changed

+161
-13
lines changed

6 files changed

+161
-13
lines changed

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java‎

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1171,10 +1171,115 @@ public SeekableByteChannel buildChannel(final String key) throws IOException {
11711171
}
11721172
}
11731173

1174+
@Test
1175+
public void TestQuestionMarkInQueryParam() throws IOException, SignatureException, XmlProcessingException {
1176+
final String bucketName = "TestQuestionMarkInQueryParam";
1177+
final String objectName = "Test?Question?Mark";
1178+
try {
1179+
HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
1180+
1181+
final List<Ds3Object> objs = Lists.newArrayList(new Ds3Object(objectName, 10));
1182+
1183+
final Ds3ClientHelpers.Job job = HELPERS.startWriteJob(bucketName, objs);
1184+
1185+
job.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
1186+
@Override
1187+
public SeekableByteChannel buildChannel(final String key) throws IOException {
1188+
1189+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(124345, 10));
1190+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
1191+
1192+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(10);
1193+
channel.write(randomBuffer);
1194+
1195+
return channel;
1196+
}
1197+
});
1198+
1199+
final GetObjectsDetailsSpectraS3Response getObjectsSpectraS3Response = client
1200+
.getObjectsDetailsSpectraS3(new GetObjectsDetailsSpectraS3Request().withName(objectName));
1201+
1202+
assertThat(getObjectsSpectraS3Response.getS3ObjectListResult().getS3Objects().size(), is(1));
1203+
1204+
} finally {
1205+
deleteAllContents(client, bucketName);
1206+
}
1207+
}
1208+
1209+
@Test
1210+
public void TestPercentInQueryParam() throws IOException, SignatureException, XmlProcessingException {
1211+
final String bucketName = "TestPercentInQueryParam";
1212+
final String objectName = "Test%Percent";
1213+
try {
1214+
HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
1215+
1216+
final List<Ds3Object> objs = Lists.newArrayList(new Ds3Object(objectName, 10));
1217+
1218+
final Ds3ClientHelpers.Job job = HELPERS.startWriteJob(bucketName, objs);
1219+
1220+
job.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
1221+
@Override
1222+
public SeekableByteChannel buildChannel(final String key) throws IOException {
1223+
1224+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(124345, 10));
1225+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
1226+
1227+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(10);
1228+
channel.write(randomBuffer);
1229+
1230+
return channel;
1231+
}
1232+
});
1233+
1234+
final GetObjectsDetailsSpectraS3Response getObjectsSpectraS3Response = client
1235+
.getObjectsDetailsSpectraS3(new GetObjectsDetailsSpectraS3Request().withName(objectName));
1236+
1237+
assertThat(getObjectsSpectraS3Response.getS3ObjectListResult().getS3Objects().size(), is(1));
1238+
1239+
} finally {
1240+
deleteAllContents(client, bucketName);
1241+
}
1242+
}
1243+
1244+
@Test
1245+
public void TestSemicolonInQueryParam() throws IOException, SignatureException, XmlProcessingException {
1246+
final String bucketName = "TestSemicolonInQueryParam";
1247+
final String objectName = "Test;Semicolon";
1248+
try {
1249+
HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
1250+
1251+
final List<Ds3Object> objs = Lists.newArrayList(new Ds3Object(objectName, 10));
1252+
1253+
final Ds3ClientHelpers.Job job = HELPERS.startWriteJob(bucketName, objs);
1254+
1255+
job.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
1256+
@Override
1257+
public SeekableByteChannel buildChannel(final String key) throws IOException {
1258+
1259+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(124345, 10));
1260+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
1261+
1262+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(10);
1263+
channel.write(randomBuffer);
1264+
1265+
return channel;
1266+
}
1267+
});
1268+
1269+
final GetObjectsDetailsSpectraS3Response getObjectsSpectraS3Response = client
1270+
.getObjectsDetailsSpectraS3(new GetObjectsDetailsSpectraS3Request().withName(objectName));
1271+
1272+
assertThat(getObjectsSpectraS3Response.getS3ObjectListResult().getS3Objects().size(), is(1));
1273+
1274+
} finally {
1275+
deleteAllContents(client, bucketName);
1276+
}
1277+
}
1278+
11741279
@Test
11751280
public void TestSpecialCharacterInObjectName() throws IOException, SignatureException, XmlProcessingException {
11761281
final String bucketName = "TestSpecialCharacterInObjectName";
1177-
final String objectName = "varsity1314/_projects/VARSITY 13-14/_versions/Varsity 13-14 (2015-10-05 1827)/_project/Trash/PC\uF022MAC HD.avb";
1282+
final String objectName = "varsity1314/_projects/VARSITY 13-14/_versions/Varsity 13-14 (2015-10-05 1827)/_project/%Tra;sh?/PC\uF022MAC HD.avb";
11781283
try {
11791284
HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
11801285

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
import com.google.common.base.Joiner;
2020
import com.google.common.collect.Iterators;
2121
import com.google.common.escape.Escaper;
22-
import com.google.common.net.UrlEscapers;
2322
import com.spectralogic.ds3client.BulkCommand;
2423
import com.spectralogic.ds3client.utils.Guard;
24+
import com.spectralogic.ds3client.utils.SafeStringManipulation;
2525

2626
import javax.annotation.Nonnull;
2727
import java.net.MalformedURLException;
@@ -48,7 +48,7 @@ public static URL buildUrl(final ConnectionDetails connectionDetails, final Stri
4848
builder.append('/');
4949
}
5050

51-
final Escaper urlEscaper = UrlEscapers.urlFragmentEscaper();
51+
final Escaper urlEscaper = SafeStringManipulation.getDs3Escaper();
5252

5353
builder.append(urlEscaper.escape(path));
5454

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
package com.spectralogic.ds3client.networking;
1717

18-
import com.google.common.net.UrlEscapers;
1918
import com.spectralogic.ds3client.Ds3InputStreamEntity;
2019
import com.spectralogic.ds3client.commands.interfaces.Ds3Request;
2120
import com.spectralogic.ds3client.exceptions.InvalidCertificate;
@@ -66,6 +65,7 @@
6665

6766
import static com.spectralogic.ds3client.utils.Signature.canonicalizeAmzHeaders;
6867
import static com.spectralogic.ds3client.utils.Signature.canonicalizeResource;
68+
import static com.spectralogic.ds3client.utils.SafeStringManipulation.getEscapedRequestPath;
6969

7070
public class NetworkClientImpl implements NetworkClient {
7171
final static private Logger LOG = LoggerFactory.getLogger(NetworkClientImpl.class);
@@ -232,7 +232,7 @@ public RequestExecutor(final CloseableHttpClient client, final HttpHost host, fi
232232
throw new RequiresMarkSupportedException();
233233
}
234234

235-
Object[] paramArray = {this.ds3Request.getVerb(), this.host.toString(), this.ds3Request.getPath()};
235+
final Object[] paramArray = {this.ds3Request.getVerb(), this.host.toString(), getEscapedRequestPath(this.ds3Request)};
236236
LOG.info("Sending request: {} {} {}", paramArray );
237237
this.checksumType = ds3Request.getChecksumType();
238238
this.hash = this.buildHash();
@@ -258,7 +258,8 @@ private HttpRequest buildHttpRequest() throws IOException {
258258
if (this.content != null) {
259259
final BasicHttpEntityEnclosingRequest httpRequest = new BasicHttpEntityEnclosingRequest(verb, path);
260260

261-
final Ds3InputStreamEntity entityStream = new Ds3InputStreamEntity(this.content, this.ds3Request.getSize(), ContentType.create(this.ds3Request.getContentType()), this.ds3Request.getPath());
261+
final Ds3InputStreamEntity entityStream = new Ds3InputStreamEntity(this.content, this.ds3Request.getSize(),
262+
ContentType.create(this.ds3Request.getContentType()), getEscapedRequestPath(this.ds3Request));
262263
entityStream.setBufferSize(NetworkClientImpl.this.connectionDetails.getBufferSize());
263264
httpRequest.setEntity(entityStream);
264265
return httpRequest;
@@ -268,7 +269,7 @@ private HttpRequest buildHttpRequest() throws IOException {
268269
}
269270

270271
private String buildPath() {
271-
String path = UrlEscapers.urlFragmentEscaper().escape(this.ds3Request.getPath());
272+
String path = getEscapedRequestPath(this.ds3Request);
272273
final Map<String, String> queryParams = this.ds3Request.getQueryParams();
273274
if (!queryParams.isEmpty()) {
274275
path += "?" + NetUtils.buildQueryString(queryParams);
@@ -301,7 +302,7 @@ private void addHeaders(final HttpRequest httpRequest) throws IOException {
301302
this.ds3Request.getContentType(),
302303
date,
303304
canonicalizeAmzHeaders(new MultiMapImpl<>(this.ds3Request.getHeaders())),
304-
canonicalizeResource(this.ds3Request.getPath(), this.ds3Request.getQueryParams()),
305+
canonicalizeResource(getEscapedRequestPath(this.ds3Request), this.ds3Request.getQueryParams()),
305306
NetworkClientImpl.this.connectionDetails.getCredentials()
306307
)));
307308
} catch (final SignatureException e) {

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/SafeStringManipulation.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@
1515

1616
package com.spectralogic.ds3client.utils;
1717

18+
import com.google.common.escape.Escaper;
19+
import com.google.common.net.PercentEscaper;
1820
import com.google.common.net.UrlEscapers;
21+
import com.spectralogic.ds3client.commands.interfaces.Ds3Request;
1922

2023
import java.util.Date;
2124

@@ -25,6 +28,14 @@
2528
*/
2629
public final class SafeStringManipulation {
2730

31+
static final String DS3_URL_PATH_FRAGMENT_SAFE_CHARS =
32+
"-._~" + // Google escaper URL_PATH_OTHER_SAFE_CHARS_LACKING_PLUS
33+
"!$'()*,&=" + // removed ; (so it will be escaped) and added / (so it will not)
34+
"@:/"; // Their urlFragmentEscaper uses URL_PATH_OTHER_SAFE_CHARS_LACKING_PLUS + "+/?"+
35+
36+
private static final Escaper DS3_URL_FRAGMENT_ESCAPER =
37+
new PercentEscaper(DS3_URL_PATH_FRAGMENT_SAFE_CHARS, false);
38+
2839
private SafeStringManipulation() {
2940
//pass
3041
}
@@ -33,7 +44,7 @@ public static <T> String safeUrlEscape(final T obj) {
3344
if (obj == null) {
3445
return null;
3546
}
36-
return UrlEscapers.urlFragmentEscaper().escape(safeToString(obj)).replace("+", "%2B");
47+
return getDs3Escaper().escape(safeToString(obj));
3748
}
3849

3950
public static <T> String safeToString(final T obj) {
@@ -48,4 +59,16 @@ public static <T> String safeToString(final T obj) {
4859
}
4960
return obj.toString();
5061
}
62+
public static Escaper getDs3Escaper() {
63+
// escaped characters in DS3 path and query parameter value segments
64+
return DS3_URL_FRAGMENT_ESCAPER;
65+
}
66+
67+
public static String getEscapedRequestPath(Ds3Request request) {
68+
if ((request == null) || (request.getPath() == null)){
69+
return "";
70+
}
71+
return getDs3Escaper().escape(request.getPath());
72+
}
73+
5174
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/Signature.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616
package com.spectralogic.ds3client.utils;
1717

1818
import com.google.common.base.Joiner;
19-
import com.google.common.collect.Multimap;
20-
import com.google.common.net.UrlEscapers;
2119
import com.spectralogic.ds3client.commands.PutObjectRequest;
2220
import com.spectralogic.ds3client.models.common.SignatureDetails;
2321

@@ -89,16 +87,19 @@ public static String signature(final SignatureDetails signatureDetails)
8987

9088
public static String canonicalizeResource(final String path, final Map<String, String> queryParams) {
9189
final StringBuilder canonicalizedResource = new StringBuilder();
92-
canonicalizedResource.append(UrlEscapers.urlFragmentEscaper().escape(path));
90+
// path is escaped
91+
canonicalizedResource.append(path);
9392

9493
if (queryParams != null) {
9594
if (queryParams.containsKey("delete")) {
9695
canonicalizedResource.append("?delete");
9796
}
98-
9997
if (queryParams.containsKey("versioning")) {
10098
canonicalizedResource.append("?versioning=").append(queryParams.get("versioning"));
10199
}
100+
if (queryParams.containsKey("uploads")) {
101+
canonicalizedResource.append("?uploads");
102+
}
102103
}
103104
return canonicalizedResource.toString();
104105
}

‎ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,24 @@ public void pathMultipleSlashes() {
7777
assertThat(result, is("/basePath/file.xml"));
7878
}
7979

80+
@Test
81+
public void pathEscapeQuestion() throws MalformedURLException {
82+
final URL result = NetUtils.buildUrl(ConnectionFixture.getConnection(), "bucket/One?Two.xml");
83+
assertThat(result.getPath(), is("/bucket/One%3FTwo.xml"));
84+
}
85+
86+
@Test
87+
public void pathEscapeSemicolon() throws MalformedURLException {
88+
final URL result = NetUtils.buildUrl(ConnectionFixture.getConnection(), "bucket/One;Two.xml");
89+
assertThat(result.getPath(), is("/bucket/One%3BTwo.xml"));
90+
}
91+
92+
@Test
93+
public void pathEscapePercent() throws MalformedURLException {
94+
final URL result = NetUtils.buildUrl(ConnectionFixture.getConnection(), "bucket/One%Two.xml");
95+
assertThat(result.getPath(), is("/bucket/One%25Two.xml"));
96+
}
97+
8098
@Test
8199
public void buildPathWithoutSlash() throws MalformedURLException {
82100
final URL result = NetUtils.buildUrl(ConnectionFixture.getConnection(), "path");

0 commit comments

Comments
 (0)