Skip to content

Commit ef115ab

Browse files
swillswill
committed
Merge pull request #1332 from syed/swift-download
Add ability to download templates in SwiftThis PR adds the ability to download templates when using Swift as a secondary storage. Uses the "temp_url" feature of Swift so that tempates can be downloaded without authenticaiton. * pr/1332: Add ability to download templates in Swift Signed-off-by: Will Stevens <[email protected]>
2 parents 4db1c01 + 52fc2ba commit ef115ab

File tree

4 files changed

+193
-13
lines changed

4 files changed

+193
-13
lines changed

‎plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,15 @@
1818
*/
1919
package org.apache.cloudstack.storage.datastore.driver;
2020

21+
import java.net.URL;
2122
import java.util.Map;
23+
import java.util.UUID;
2224

2325
import javax.inject.Inject;
2426

27+
import com.cloud.configuration.Config;
28+
import com.cloud.utils.SwiftUtil;
29+
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
2530
import org.apache.log4j.Logger;
2631

2732
import org.apache.cloudstack.api.ApiConstants;
@@ -43,7 +48,6 @@
4348
import com.cloud.agent.api.to.DataObjectType;
4449
import com.cloud.agent.api.to.DataStoreTO;
4550
import com.cloud.agent.api.to.SwiftTO;
46-
import com.cloud.exception.UnsupportedServiceException;
4751
import com.cloud.storage.Storage.ImageFormat;
4852
import com.cloud.template.VirtualMachineTemplate;
4953
import com.cloud.utils.exception.CloudRuntimeException;
@@ -57,6 +61,8 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
5761
EndPointSelector _epSelector;
5862
@Inject
5963
StorageCacheManager cacheManager;
64+
@Inject
65+
ConfigurationDao _configDao;
6066

6167
@Override
6268
public DataStoreTO getStoreTO(DataStore store) {
@@ -67,7 +73,29 @@ public DataStoreTO getStoreTO(DataStore store) {
6773

6874
@Override
6975
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
70-
throw new UnsupportedServiceException("Extract entity url is not yet supported for Swift image store provider");
76+
77+
SwiftTO swiftTO = (SwiftTO)store.getTO();
78+
String tempKey = UUID.randomUUID().toString();
79+
boolean result = SwiftUtil.setTempKey(swiftTO, tempKey);
80+
81+
if (!result) {
82+
String errMsg = "Unable to set Temp-Key: " + tempKey;
83+
s_logger.error(errMsg);
84+
throw new CloudRuntimeException(errMsg);
85+
}
86+
87+
String containerName = SwiftUtil.getContainerName(dataObject.getType().toString(), dataObject.getId());
88+
String objectName = installPath.split("\\/")[1];
89+
// Get extract url expiration interval set in global configuration (in seconds)
90+
int urlExpirationInterval = Integer.parseInt(_configDao.getValue(Config.ExtractURLExpirationInterval.toString()));
91+
92+
URL swiftUrl = SwiftUtil.generateTempUrl(swiftTO, containerName, objectName, tempKey, urlExpirationInterval);
93+
if (swiftUrl != null) {
94+
s_logger.debug("Swift temp-url: " + swiftUrl.toString());
95+
return swiftUrl.toString();
96+
}
97+
98+
throw new CloudRuntimeException("Unable to create extraction URL");
7199
}
72100

73101
@Override

‎server/src/com/cloud/template/TemplateManagerImpl.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1520,12 +1520,6 @@ public VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) t
15201520
}
15211521

15221522
} finally {
1523-
/*if (snapshot != null && snapshot.getSwiftId() != null
1524-
&& secondaryStorageURL != null && zoneId != null
1525-
&& accountId != null && volumeId != null) {
1526-
_snapshotMgr.deleteSnapshotsForVolume(secondaryStorageURL,
1527-
zoneId, accountId, volumeId);
1528-
}*/
15291523
if (privateTemplate == null) {
15301524
final VolumeVO volumeFinal = volume;
15311525
final SnapshotVO snapshotFinal = snapshot;

‎utils/src/main/java/com/cloud/utils/SwiftUtil.java‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,30 @@
2020
package com.cloud.utils;
2121

2222
import java.io.File;
23-
import java.util.Arrays;
23+
import java.net.URL;
24+
import java.security.InvalidKeyException;
25+
import java.security.NoSuchAlgorithmException;
26+
import java.security.SignatureException;
2427
import java.util.Map;
28+
import java.util.Arrays;
29+
import java.util.HashMap;
30+
import java.util.Formatter;
2531

2632
import org.apache.log4j.Logger;
2733

2834
import com.cloud.utils.exception.CloudRuntimeException;
2935
import com.cloud.utils.script.OutputInterpreter;
3036
import com.cloud.utils.script.Script;
3137

38+
import javax.crypto.Mac;
39+
import javax.crypto.spec.SecretKeySpec;
40+
3241
public class SwiftUtil {
3342
private static Logger logger = Logger.getLogger(SwiftUtil.class);
3443
private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
44+
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
45+
46+
3547

3648
public interface SwiftClientCfg {
3749
String getAccount();
@@ -143,8 +155,7 @@ public static String[] list(SwiftClientCfg swift, String container, String rFile
143155
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
144156
String result = command.execute(parser);
145157
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
146-
String[] lines = parser.getLines().split("\\n");
147-
return lines;
158+
return parser.getLines().split("\\n");
148159
} else {
149160
if (result != null) {
150161
String errMsg = "swiftList failed , err=" + result;
@@ -161,7 +172,7 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
161172
int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
162173
String container = swiftPath.substring(0, firstIndexOfSeparator);
163174
String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
164-
String destFilePath = null;
175+
String destFilePath;
165176
if (destDirectory.isDirectory()) {
166177
destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
167178
} else {
@@ -171,7 +182,7 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
171182
Script command = new Script("/bin/bash", logger);
172183
command.add("-c");
173184
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
174-
" download " + container + " " + srcPath + " -o " + destFilePath);
185+
" download " + container + " " + srcPath + " -o " + destFilePath);
175186
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
176187
String result = command.execute(parser);
177188
if (result != null) {
@@ -236,4 +247,59 @@ public static boolean deleteObject(SwiftClientCfg cfg, String path) {
236247
command.execute(parser);
237248
return true;
238249
}
250+
251+
public static boolean setTempKey(SwiftClientCfg cfg, String tempKey){
252+
253+
Map<String, String> tempKeyMap = new HashMap<>();
254+
tempKeyMap.put("Temp-URL-Key", tempKey);
255+
return postMeta(cfg, "", "", tempKeyMap);
256+
257+
}
258+
259+
public static URL generateTempUrl(SwiftClientCfg cfg, String container, String object, String tempKey, int urlExpirationInterval) {
260+
261+
int currentTime = (int) (System.currentTimeMillis() / 1000L);
262+
int expirationSeconds = currentTime + urlExpirationInterval;
263+
264+
try {
265+
266+
URL endpoint = new URL(cfg.getEndPoint());
267+
String method = "GET";
268+
String path = String.format("/v1/AUTH_%s/%s/%s", cfg.getAccount(), container, object);
269+
270+
//sign the request
271+
String hmacBody = String.format("%s\n%d\n%s", method, expirationSeconds, path);
272+
String signature = calculateRFC2104HMAC(hmacBody, tempKey);
273+
path += String.format("?temp_url_sig=%s&temp_url_expires=%d", signature, expirationSeconds);
274+
275+
//generate the temp url
276+
URL tempUrl = new URL(endpoint.getProtocol(), endpoint.getHost(), endpoint.getPort(), path);
277+
278+
return tempUrl;
279+
280+
} catch (Exception e) {
281+
logger.error(e.getMessage());
282+
throw new CloudRuntimeException(e.getMessage());
283+
}
284+
285+
}
286+
287+
public static String calculateRFC2104HMAC(String data, String key)
288+
throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
289+
290+
SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
291+
Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM);
292+
mac.init(signingKey);
293+
return toHexString(mac.doFinal(data.getBytes()));
294+
295+
}
296+
297+
public static String toHexString(byte[] bytes) {
298+
299+
Formatter formatter = new Formatter();
300+
for (byte b : bytes) {
301+
formatter.format("%02x", b);
302+
}
303+
return formatter.toString();
304+
}
239305
}

‎utils/src/test/java/com/cloud/utils/SwiftUtilTest.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package com.cloud.utils;
21+
22+
import org.junit.Test;
23+
import org.mockito.Mockito;
24+
25+
import java.net.URL;
26+
import java.security.InvalidKeyException;
27+
import java.security.NoSuchAlgorithmException;
28+
import java.security.SignatureException;
29+
30+
import static org.junit.Assert.assertArrayEquals;
31+
import static org.junit.Assert.assertEquals;
32+
import static org.junit.Assert.assertTrue;
33+
import static org.mockito.Mockito.when;
34+
35+
36+
public class SwiftUtilTest {
37+
38+
@Test
39+
public void testCalculateRFC2104HMAC() throws NoSuchAlgorithmException, SignatureException, InvalidKeyException {
40+
String inputData = "testData";
41+
String inputKey = "testKey";
42+
String expected = "1d541ecb5cdb2d850716bfd55585e20a1cd8984b";
43+
String output = SwiftUtil.calculateRFC2104HMAC(inputData, inputKey);
44+
45+
assertEquals(expected, output);
46+
}
47+
48+
@Test
49+
public void testToHexString(){
50+
final byte[] input = "testing".getBytes();
51+
final String expected = "74657374696e67";
52+
final String result = SwiftUtil.toHexString(input);
53+
assertEquals(expected, result);
54+
}
55+
56+
@Test
57+
public void testGenerateTempUrl() {
58+
59+
SwiftUtil.SwiftClientCfg cfg = Mockito.mock(SwiftUtil.SwiftClientCfg.class);
60+
when(cfg.getEndPoint()).thenReturn("http://localhost:8080/v1/");
61+
when(cfg.getAccount()).thenReturn("test");
62+
63+
String container = "testContainer";
64+
String object = "testObject";
65+
String tempKey = "testKey";
66+
int urlExpirationInterval = 3600;
67+
String expected = "http://localhost:8080/v1/AUTH_test/testContainer/testObject";
68+
URL output = SwiftUtil.generateTempUrl(cfg, container, object, tempKey, urlExpirationInterval);
69+
70+
assertTrue(output.toString().contains(expected));
71+
}
72+
73+
@Test
74+
public void testSplitSwiftPath(){
75+
String input = "container/object";
76+
String[] output = SwiftUtil.splitSwiftPath(input);
77+
String[] expected = {"container", "object"};
78+
79+
assertArrayEquals(expected, output);
80+
}
81+
82+
@Test
83+
public void testGetContainerName(){
84+
85+
String inputType = "Template";
86+
long inputId = 45;
87+
String output = SwiftUtil.getContainerName(inputType, inputId);
88+
String expected = "T-45";
89+
90+
assertEquals(expected, output);
91+
}
92+
}

0 commit comments

Comments
 (0)