Skip to content

Commit 977f6b6

Browse files
nvazqueznvazquez
committed
Merge branch 'main' into 422-vmware-to-kvm-improvements
2 parents 2c52bfa + 09b63bc commit 977f6b6

File tree

28 files changed

+901
-23
lines changed

28 files changed

+901
-23
lines changed

‎api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,19 +77,24 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
7777
@Param(description = "the name of the cluster for the storage pool")
7878
private String clusterName;
7979

80+
@SerializedName(ApiConstants.CAPACITY_BYTES)
81+
@Param(description = "bytes CloudStack can provision from this storage pool", since = "4.22.0")
82+
private Long capacityBytes;
83+
84+
@Deprecated(since = "4.22.0")
8085
@SerializedName("disksizetotal")
8186
@Param(description = "the total disk size of the storage pool")
8287
private Long diskSizeTotal;
8388

8489
@SerializedName("disksizeallocated")
85-
@Param(description = "the host's currently allocated disk size")
90+
@Param(description = "the pool's currently allocated disk size")
8691
private Long diskSizeAllocated;
8792

8893
@SerializedName("disksizeused")
89-
@Param(description = "the host's currently used disk size")
94+
@Param(description = "the pool's currently used disk size")
9095
private Long diskSizeUsed;
9196

92-
@SerializedName("capacityiops")
97+
@SerializedName(ApiConstants.CAPACITY_IOPS)
9398
@Param(description = "IOPS CloudStack can provision from this storage pool")
9499
private Long capacityIops;
95100

@@ -288,6 +293,14 @@ public void setClusterName(String clusterName) {
288293
this.clusterName = clusterName;
289294
}
290295

296+
public Long getCapacityBytes() {
297+
return capacityBytes;
298+
}
299+
300+
public void setCapacityBytes(Long capacityBytes) {
301+
this.capacityBytes = capacityBytes;
302+
}
303+
291304
public Long getDiskSizeTotal() {
292305
return diskSizeTotal;
293306
}

‎api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,23 @@
2222

2323
import com.cloud.utils.component.Manager;
2424

25+
import java.io.IOException;
26+
2527
public interface UserDataManager extends Manager, Configurable {
2628
String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length";
2729
ConfigKey<Integer> VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768",
2830
"Max length of vm userdata after base64 encoding. Default is 32768 and maximum is 1048576", true);
2931

3032
String concatenateUserData(String userdata1, String userdata2, String userdataProvider);
3133
String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod);
34+
35+
/**
36+
* This method validates the user data uuid for system VMs and returns the user data
37+
* after compression and base64 encoding for the system VM to consume.
38+
*
39+
* @param userDataUuid
40+
* @return a String containing the user data after compression and base64 encoding
41+
* @throws IOException
42+
*/
43+
String validateAndGetUserDataForSystemVM(String userDataUuid) throws IOException;
3244
}

‎engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ public interface VirtualMachineManager extends Manager {
106106
ConfigKey<Boolean> VmSyncPowerStateTransitioning = new ConfigKey<>("Advanced", Boolean.class, "vm.sync.power.state.transitioning", "true",
107107
"Whether to sync power states of the transitioning and stalled VMs while processing VM power reports.", false);
108108

109+
ConfigKey<Boolean> SystemVmEnableUserData = new ConfigKey<>(Boolean.class, "systemvm.userdata.enabled", "Advanced", "false",
110+
"Enable user data for system VMs. When enabled, the CPVM, SSVM, and Router system VMs will use the values from the global settings console.proxy.vm.userdata, secstorage.vm.userdata, and virtual.router.userdata, respectively, to provide cloud-init user data to the VM.",
111+
true, ConfigKey.Scope.Zone, null);
109112

110113
interface Topics {
111114
String VM_POWER_STATE = "vm.powerstate";

‎engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreLifeCycle.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@
2424
import com.cloud.storage.StoragePool;
2525

2626
public interface PrimaryDataStoreLifeCycle extends DataStoreLifeCycle {
27-
public static final String CAPACITY_BYTES = "capacityBytes";
28-
public static final String CAPACITY_IOPS = "capacityIops";
27+
String CAPACITY_BYTES = "capacityBytes";
28+
String CAPACITY_IOPS = "capacityIops";
2929

3030
void updateStoragePool(StoragePool storagePool, Map<String, String> details);
3131
void enableStoragePool(DataStore store);

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5244,7 +5244,7 @@ public ConfigKey<?>[] getConfigKeys() {
52445244
VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
52455245
HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
52465246
AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName,
5247-
VmSyncPowerStateTransitioning
5247+
VmSyncPowerStateTransitioning, SystemVmEnableUserData
52485248
};
52495249
}
52505250

‎engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,9 @@ public StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, Li
320320
pool = super.persist(pool);
321321
if (details != null) {
322322
for (Map.Entry<String, String> detail : details.entrySet()) {
323+
if (detail.getKey().toLowerCase().contains("password") || detail.getKey().toLowerCase().contains("token")) {
324+
displayDetails = false;
325+
}
323326
StoragePoolDetailVO vo = new StoragePoolDetailVO(pool.getId(), detail.getKey(), detail.getValue(), displayDetails);
324327
_detailsDao.persist(vo);
325328
}

‎engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid
3535
-- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository.
3636
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone''');
3737

38+
-- Updated display to false for password/token detail of the storage pool details
39+
UPDATE `cloud`.`storage_pool_details` SET display = 0 WHERE name LIKE '%password%';
40+
UPDATE `cloud`.`storage_pool_details` SET display = 0 WHERE name LIKE '%token%';
41+
3842
-- VMware to KVM migration improvements
3943
CREATE TABLE IF NOT EXISTS `cloud`.`import_vm_task`(
4044
`id` bigint unsigned NOT NULL auto_increment COMMENT 'id',

‎engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,7 @@ public class PrimaryDataStoreHelper {
8585
DataStoreProviderManager dataStoreProviderMgr;
8686

8787
public DataStore createPrimaryDataStore(PrimaryDataStoreParameters params) {
88-
if(params == null)
89-
{
88+
if (params == null) {
9089
throw new InvalidParameterValueException("createPrimaryDataStore: Input params is null, please check");
9190
}
9291
StoragePoolVO dataStoreVO = dataStoreDao.findPoolByUUID(params.getUuid());

‎engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,18 @@
1616
// under the License.
1717
package org.apache.cloudstack.userdata;
1818

19+
import java.io.IOException;
1920
import java.io.UnsupportedEncodingException;
2021
import java.net.URLDecoder;
2122
import java.util.HashMap;
2223
import java.util.List;
2324
import java.util.Map;
2425

26+
import com.cloud.domain.Domain;
27+
import com.cloud.user.User;
28+
import com.cloud.user.UserDataVO;
29+
import com.cloud.user.dao.UserDataDao;
30+
import com.cloud.utils.compression.CompressionUtil;
2531
import org.apache.cloudstack.api.BaseCmd;
2632
import org.apache.cloudstack.framework.config.ConfigKey;
2733
import org.apache.commons.codec.binary.Base64;
@@ -31,7 +37,12 @@
3137
import com.cloud.utils.component.ManagerBase;
3238
import com.cloud.utils.exception.CloudRuntimeException;
3339

40+
import javax.inject.Inject;
41+
3442
public class UserDataManagerImpl extends ManagerBase implements UserDataManager {
43+
@Inject
44+
UserDataDao userDataDao;
45+
3546
private static final int MAX_USER_DATA_LENGTH_BYTES = 2048;
3647
private static final int MAX_HTTP_GET_LENGTH = 2 * MAX_USER_DATA_LENGTH_BYTES; // 4KB
3748
private static final int NUM_OF_2K_BLOCKS = 512;
@@ -118,6 +129,25 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) {
118129
return Base64.encodeBase64String(decodedUserData);
119130
}
120131

132+
@Override
133+
public String validateAndGetUserDataForSystemVM(String userDataUuid) throws IOException {
134+
if (StringUtils.isBlank(userDataUuid)) {
135+
return null;
136+
}
137+
UserDataVO userDataVo = userDataDao.findByUuid(userDataUuid);
138+
if (userDataVo == null) {
139+
return null;
140+
}
141+
if (userDataVo.getDomainId() == Domain.ROOT_DOMAIN && userDataVo.getAccountId() == User.UID_ADMIN) {
142+
// Decode base64 user data, compress it, then re-encode to reduce command line length
143+
String plainTextUserData = new String(java.util.Base64.getDecoder().decode(userDataVo.getUserData()));
144+
CompressionUtil compressionUtil = new CompressionUtil();
145+
byte[] compressedUserData = compressionUtil.compressString(plainTextUserData);
146+
return java.util.Base64.getEncoder().encodeToString(compressedUserData);
147+
}
148+
throw new CloudRuntimeException("User data can only be used by system VMs if it belongs to the ROOT domain and ADMIN account.");
149+
}
150+
121151
private byte[] validateAndDecodeByHTTPMethod(String userData, int maxHTTPLength, BaseCmd.HTTPMethod httpMethod) {
122152
byte[] decodedUserData = Base64.decodeBase64(userData.getBytes());
123153
if (decodedUserData == null || decodedUserData.length < 1) {

‎engine/userdata/src/test/java/org/apache/cloudstack/userdata/UserDataManagerImplTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,37 @@
1717
package org.apache.cloudstack.userdata;
1818

1919
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertFalse;
21+
import static org.junit.Assert.assertNotEquals;
22+
import static org.junit.Assert.assertNotNull;
23+
import static org.junit.Assert.assertNull;
24+
import static org.mockito.Mockito.when;
2025

26+
import java.io.IOException;
2127
import java.nio.charset.StandardCharsets;
28+
import java.util.Base64;
2229

2330
import org.apache.cloudstack.api.BaseCmd;
2431
import org.junit.Test;
2532
import org.junit.runner.RunWith;
2633
import org.mockito.InjectMocks;
34+
import org.mockito.Mock;
35+
import org.mockito.Mockito;
2736
import org.mockito.Spy;
2837
import org.mockito.junit.MockitoJUnitRunner;
2938

39+
import com.cloud.domain.Domain;
40+
import com.cloud.user.User;
41+
import com.cloud.user.UserDataVO;
42+
import com.cloud.user.dao.UserDataDao;
43+
import com.cloud.utils.exception.CloudRuntimeException;
44+
3045
@RunWith(MockitoJUnitRunner.class)
3146
public class UserDataManagerImplTest {
3247

48+
@Mock
49+
private UserDataDao userDataDao;
50+
3351
@Spy
3452
@InjectMocks
3553
private UserDataManagerImpl userDataManager;
@@ -56,4 +74,76 @@ public void testValidateUrlEncodedBase64() {
5674
assertEquals("validate return the value with padding", encodedUserdata, userDataManager.validateUserData(urlEncodedUserdata, BaseCmd.HTTPMethod.GET));
5775
}
5876

77+
@Test
78+
public void testValidateAndGetUserDataForSystemVMWithBlankUuid() throws IOException {
79+
// Test with blank UUID should return null
80+
assertNull("null UUID should return null", userDataManager.validateAndGetUserDataForSystemVM(null));
81+
assertNull("blank UUID should return null", userDataManager.validateAndGetUserDataForSystemVM(""));
82+
assertNull("blank UUID should return null", userDataManager.validateAndGetUserDataForSystemVM(" "));
83+
}
84+
85+
@Test
86+
public void testValidateAndGetUserDataForSystemVMNotFound() throws IOException {
87+
// Test when userDataVo is not found
88+
String testUuid = "test-uuid-123";
89+
when(userDataDao.findByUuid(testUuid)).thenReturn(null);
90+
91+
assertNull("userdata not found should return null", userDataManager.validateAndGetUserDataForSystemVM(testUuid));
92+
}
93+
94+
@Test(expected = CloudRuntimeException.class)
95+
public void testValidateAndGetUserDataForSystemVMInvalidDomain() throws IOException {
96+
// Test with userDataVo that doesn't belong to ROOT domain
97+
String testUuid = "test-uuid-123";
98+
UserDataVO userDataVo = Mockito.mock(UserDataVO.class);
99+
when(userDataVo.getDomainId()).thenReturn(2L); // Not ROOT domain
100+
101+
when(userDataDao.findByUuid(testUuid)).thenReturn(userDataVo);
102+
userDataManager.validateAndGetUserDataForSystemVM(testUuid);
103+
}
104+
105+
@Test(expected = CloudRuntimeException.class)
106+
public void testValidateAndGetUserDataForSystemVMInvalidAccount() throws IOException {
107+
// Test with userDataVo that doesn't belong to ADMIN account
108+
String testUuid = "test-uuid-123";
109+
UserDataVO userDataVo = Mockito.mock(UserDataVO.class);
110+
when(userDataVo.getDomainId()).thenReturn(Domain.ROOT_DOMAIN);
111+
when(userDataVo.getAccountId()).thenReturn(3L);
112+
userDataVo.setUserData("dGVzdCBkYXRh"); // "test data" in base64
113+
114+
when(userDataDao.findByUuid(testUuid)).thenReturn(userDataVo);
115+
userDataManager.validateAndGetUserDataForSystemVM(testUuid);
116+
}
117+
118+
@Test
119+
public void testValidateAndGetUserDataForSystemVMValidSystemVMUserData() throws IOException {
120+
// Test with valid system VM userdata (ROOT domain + ADMIN account)
121+
String testUuid = "test-uuid-123";
122+
String originalText = "#!/bin/bash\necho 'Hello World'";
123+
String base64EncodedUserData = Base64.getEncoder().encodeToString(originalText.getBytes());
124+
125+
UserDataVO userDataVo = Mockito.mock(UserDataVO.class);
126+
when(userDataVo.getDomainId()).thenReturn(Domain.ROOT_DOMAIN);
127+
when(userDataVo.getAccountId()).thenReturn(User.UID_ADMIN);
128+
when(userDataVo.getUserData()).thenReturn(base64EncodedUserData);
129+
130+
when(userDataDao.findByUuid(testUuid)).thenReturn(userDataVo);
131+
132+
String result = userDataManager.validateAndGetUserDataForSystemVM(testUuid);
133+
134+
// Verify result is not null and is base64 encoded
135+
assertNotNull("result should not be null", result);
136+
assertFalse("result should be base64 encoded", result.isEmpty());
137+
138+
// Verify the result is valid base64
139+
try {
140+
Base64.getDecoder().decode(result);
141+
} catch (IllegalArgumentException e) {
142+
throw new AssertionError("Result should be valid base64", e);
143+
}
144+
145+
// The result should be different from input since it's compressed
146+
assertNotEquals("compressed result should be different from original", result, base64EncodedUserData);
147+
}
148+
59149
}

0 commit comments

Comments
 (0)