Skip to content

Commit d8e4db0

Browse files
WikiDruWikiDru
committed
[#1] Support AWS S3
1 parent 4e5be1e commit d8e4db0

File tree

7 files changed

+121
-27
lines changed

7 files changed

+121
-27
lines changed

‎thumbly-http-client/src/main/java/org/code13k/thumbly/web/client/CachedWebClient.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33

44
import org.apache.commons.codec.digest.DigestUtils;
5-
import org.apache.commons.io.FileUtils;
65
import org.apache.commons.io.FilenameUtils;
76
import org.apache.commons.lang3.StringUtils;
7+
import org.code13k.thumbly.web.client.aws.AwsS3SignValue;
88
import org.code13k.thumbly.web.client.model.WebData;
99
import org.slf4j.Logger;
1010
import org.slf4j.LoggerFactory;
@@ -22,10 +22,6 @@ public class CachedWebClient {
2222
// Logger
2323
private static final Logger mLogger = LoggerFactory.getLogger(CachedWebClient.class);
2424

25-
// Const
26-
private static final String DEFAULT_CACHE_DIRECTORY = ".cache";
27-
private static final long DEFAULT_MAX_SIZE_OF_CACHE_DIRECTORY = 1024 * 1024 * 1024 * 1; // 1GB
28-
2925
// Data
3026
private Map<String, List<Consumer<String>>> mRequestMap = null;
3127
private WebRequest mWebRequest = null;
@@ -66,7 +62,7 @@ synchronized public void init(String cacheDirectory, long maxSizeOfCacheDirector
6662
/**
6763
* Get file
6864
*/
69-
public void getFile(String url, Map<String, String> headers, Consumer<String> consumer) {
65+
public void getFile(String url, AwsS3SignValue awsS3SignValue, Consumer<String> consumer) {
7066
// Check if duplicate request is already running
7167
final String key = generateKey(url);
7268
mLogger.trace("key=" + key);
@@ -84,7 +80,7 @@ public void getFile(String url, Map<String, String> headers, Consumer<String> co
8480
}
8581

8682
// Request
87-
mWebRequest.request(url, headers, new Consumer<String>() {
83+
mWebRequest.request(url, awsS3SignValue, new Consumer<String>() {
8884
@Override
8985
public void accept(String filePath) {
9086
List<Consumer<String>> consumerList = null;

‎thumbly-http-client/src/main/java/org/code13k/thumbly/web/client/WebRequest.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import io.vertx.ext.web.client.WebClientOptions;
1010
import org.apache.commons.lang3.StringUtils;
1111
import org.apache.commons.lang3.math.NumberUtils;
12+
import org.code13k.thumbly.web.client.aws.AwsS3SignValue;
13+
import org.code13k.thumbly.web.client.aws.AwsS3SignerV4;
1214
import org.code13k.thumbly.web.client.model.WebData;
1315
import org.code13k.thumbly.web.client.model.ResponseHeaders;
1416
import org.slf4j.Logger;
@@ -17,7 +19,6 @@
1719
import java.nio.file.Files;
1820
import java.nio.file.Paths;
1921
import java.util.Date;
20-
import java.util.Map;
2122
import java.util.function.Consumer;
2223

2324
public class WebRequest {
@@ -52,7 +53,7 @@ public WebRequest(int eventLoopPoolSize) {
5253
/**
5354
* Request
5455
*/
55-
public void request(String url, Map<String, String> headers, Consumer<String> consumer) {
56+
public void request(String url, AwsS3SignValue awsS3SignValue, Consumer<String> consumer) {
5657
final long startTime = System.currentTimeMillis();
5758
final String urlKey = CachedWebClient.generateKey(url);
5859
WebData tempWebData = WebDataStore.getInstance().get(urlKey);
@@ -98,11 +99,9 @@ public void request(String url, Map<String, String> headers, Consumer<String> co
9899
}
99100
}
100101

101-
// Set Headers (Custom)
102-
if (headers != null) {
103-
headers.forEach((key, value) -> {
104-
httpRequest.putHeader(key, value);
105-
});
102+
// AWS S3 Signer
103+
if (awsS3SignValue != null) {
104+
AwsS3SignerV4.putHeaders(httpRequest.headers(), url, awsS3SignValue);
106105
}
107106

108107
// Log

‎thumbly-http-client/src/main/java/org/code13k/thumbly/web/client/aws/AwsS3SignValue.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package org.code13k.thumbly.web.client.aws;
2+
3+
public class AwsS3SignValue {
4+
private String accessKey;
5+
private String secretKey;
6+
private String region;
7+
8+
public String getAccessKey() {
9+
return accessKey;
10+
}
11+
12+
public void setAccessKey(String accessKey) {
13+
this.accessKey = accessKey;
14+
}
15+
16+
public String getSecretKey() {
17+
return secretKey;
18+
}
19+
20+
public void setSecretKey(String secretKey) {
21+
this.secretKey = secretKey;
22+
}
23+
24+
public String getRegion() {
25+
return region;
26+
}
27+
28+
public void setRegion(String region) {
29+
this.region = region;
30+
}
31+
}

‎thumbly-http-client/src/main/java/org/code13k/thumbly/web/client/aws/AwsS3SignerV4.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package org.code13k.thumbly.web.client.aws;
2+
3+
import io.vertx.core.MultiMap;
4+
import org.slf4j.Logger;
5+
import org.slf4j.LoggerFactory;
6+
import uk.co.lucasweb.aws.v4.signer.HttpRequest;
7+
import uk.co.lucasweb.aws.v4.signer.Signer;
8+
import uk.co.lucasweb.aws.v4.signer.credentials.AwsCredentials;
9+
10+
import java.net.URI;
11+
import java.text.SimpleDateFormat;
12+
import java.util.TimeZone;
13+
14+
public class AwsS3SignerV4 {
15+
// Logger
16+
private static final Logger mLogger = LoggerFactory.getLogger(AwsS3SignerV4.class);
17+
18+
// Const
19+
private static final String CONTENT_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
20+
private static final String AMAZON_DATE_FORMAT = "yyyyMMdd'T'HHmmss'Z'";
21+
22+
/**
23+
* Put headers
24+
*/
25+
public static boolean putHeaders(MultiMap headers, String url, AwsS3SignValue awsS3SignValue) {
26+
try {
27+
// Init
28+
URI uri = new URI(url);
29+
HttpRequest request = new HttpRequest("GET", uri);
30+
Signer.Builder signerBuilder = Signer.builder();
31+
32+
// Put Headers
33+
headers.add("Host", uri.getHost());
34+
headers.add("X-Amz-Date", getCurrentDate());
35+
headers.add("X-Amz-Content-Sha256", CONTENT_SHA256);
36+
37+
// Set Data For Sign
38+
signerBuilder.awsCredentials(new AwsCredentials(awsS3SignValue.getAccessKey(), awsS3SignValue.getSecretKey()));
39+
signerBuilder.region(awsS3SignValue.getRegion());
40+
headers.forEach(header -> {
41+
signerBuilder.header(header.getKey(), header.getValue());
42+
});
43+
44+
// Generate authorization string
45+
String authorization = signerBuilder.buildS3(request, CONTENT_SHA256).getSignature();
46+
47+
// Result
48+
headers.add("Authorization", authorization);
49+
return true;
50+
} catch (Exception e) {
51+
mLogger.error("Failed to put headers for s3", e);
52+
return false;
53+
}
54+
}
55+
56+
/**
57+
* getCurrentDate()
58+
*/
59+
private static String getCurrentDate() {
60+
SimpleDateFormat format = new SimpleDateFormat(AMAZON_DATE_FORMAT);
61+
format.setTimeZone(TimeZone.getTimeZone("UTC"));
62+
String result = format.format(System.currentTimeMillis());
63+
return result;
64+
}
65+
66+
}

‎thumbly-main/src/main/java/org/code13k/thumbly/service/api/controller/BasicAPI.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package org.code13k.thumbly.service.api.controller;
22

33
import com.google.gson.GsonBuilder;
4-
import io.vertx.core.json.JsonObject;
54
import org.apache.commons.lang3.StringUtils;
65

76
import java.util.HashMap;

‎thumbly-main/src/main/java/org/code13k/thumbly/service/api/controller/CacheAPI.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import org.code13k.thumbly.web.client.CachedWebClient;
55
import org.code13k.thumbly.web.client.model.WebData;
66

7-
import java.util.HashMap;
87
import java.util.Map;
98

109
public class CacheAPI extends BasicAPI {

‎thumbly-main/src/main/java/org/code13k/thumbly/service/main/MainHttpServer.java‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,20 @@
88
import org.code13k.thumbly.image.processor.CachedImageProcessor;
99
import org.code13k.thumbly.image.processor.model.Command;
1010
import org.code13k.thumbly.lib.Util;
11+
import org.code13k.thumbly.model.config.channel.AwsS3Info;
1112
import org.code13k.thumbly.web.client.CachedWebClient;
1213
import org.code13k.thumbly.model.config.channel.ChannelInfo;
13-
import io.netty.handler.codec.http.HttpHeaderNames;
1414
import io.vertx.core.AbstractVerticle;
1515
import io.vertx.core.http.HttpServer;
1616
import io.vertx.core.http.HttpServerOptions;
1717
import io.vertx.ext.web.Router;
1818
import io.vertx.ext.web.RoutingContext;
1919
import org.apache.commons.lang3.StringUtils;
20+
import org.code13k.thumbly.web.client.aws.AwsS3SignValue;
2021
import org.slf4j.Logger;
2122
import org.slf4j.LoggerFactory;
2223

23-
import java.io.File;
24-
import java.net.FileNameMap;
25-
import java.net.URLConnection;
26-
import java.text.SimpleDateFormat;
2724
import java.util.ArrayList;
28-
import java.util.Date;
29-
import java.util.List;
30-
import java.util.TimeZone;
3125
import java.util.function.Consumer;
3226

3327

@@ -213,7 +207,6 @@ private void handler(RoutingContext routingContext,
213207
}
214208
}
215209

216-
217210
/**
218211
* Check command
219212
*/
@@ -239,7 +232,6 @@ private void handler(RoutingContext routingContext,
239232
String originUrl = channelInfo.getBaseUrl() + "/" + pathString;
240233
mLogger.debug("originUrl = " + originUrl);
241234

242-
243235
/**
244236
* Get status
245237
*/
@@ -280,10 +272,22 @@ private void handler(RoutingContext routingContext,
280272
* Get image
281273
*/
282274
else {
275+
/**
276+
* AWS S3
277+
*/
278+
AwsS3SignValue awsS3SignValue = null;
279+
if (channelInfo.getType() == ChannelConfig.ChannelType.AWS_S3) {
280+
AwsS3Info awsS3Info = (AwsS3Info) channelInfo;
281+
awsS3SignValue = new AwsS3SignValue();
282+
awsS3SignValue.setAccessKey(awsS3Info.getAccessKey());
283+
awsS3SignValue.setSecretKey(awsS3Info.getSecretKey());
284+
awsS3SignValue.setRegion(awsS3Info.getRegion());
285+
}
286+
283287
/**
284288
* Get origin file
285289
*/
286-
CachedWebClient.getInstance().getFile(originUrl, null, new Consumer<String>() {
290+
CachedWebClient.getInstance().getFile(originUrl, awsS3SignValue, new Consumer<String>() {
287291
@Override
288292
public void accept(String originFilePath) {
289293
mLogger.debug("originFilePath = " + originFilePath);

0 commit comments

Comments
 (0)