Skip to content

Commit bf7ae51

Browse files
Pearl1594Pearl1594
committed
Merge branch '4.19' of https://github.com/apache/cloudstack into add-ip-range-checks-sharednet
2 parents 524fff4 + f652ad0 commit bf7ae51

File tree

36 files changed

+427
-627
lines changed

36 files changed

+427
-627
lines changed

‎api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java‎

Lines changed: 1 addition & 430 deletions
Large diffs are not rendered by default.

‎client/conf/server.properties.in‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ session.timeout=30
3232
# Max allowed API request payload/content size in bytes
3333
request.content.size=1048576
3434

35+
# Max allowed API request form keys
36+
request.max.form.keys=5000
37+
3538
# Options to configure and enable HTTPS on the management server
3639
#
3740
# For the management server to pick up these configuration settings, the configured

‎client/src/main/java/org/apache/cloudstack/ServerDaemon.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ public class ServerDaemon implements Daemon {
8181
private static final String ACCESS_LOG = "access.log";
8282
private static final String REQUEST_CONTENT_SIZE_KEY = "request.content.size";
8383
private static final int DEFAULT_REQUEST_CONTENT_SIZE = 1048576;
84+
private static final String REQUEST_MAX_FORM_KEYS_KEY = "request.max.form.keys";
85+
private static final int DEFAULT_REQUEST_MAX_FORM_KEYS = 5000;
8486

8587
////////////////////////////////////////////////////////
8688
/////////////// Server Configuration ///////////////////
@@ -93,6 +95,7 @@ public class ServerDaemon implements Daemon {
9395
private int httpsPort = 8443;
9496
private int sessionTimeout = 30;
9597
private int maxFormContentSize = DEFAULT_REQUEST_CONTENT_SIZE;
98+
private int maxFormKeys = DEFAULT_REQUEST_MAX_FORM_KEYS;
9699
private boolean httpsEnable = false;
97100
private String accessLogFile = "access.log";
98101
private String bindInterface = null;
@@ -140,6 +143,7 @@ public void init(final DaemonContext context) {
140143
setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));
141144
setSessionTimeout(Integer.valueOf(properties.getProperty(SESSION_TIMEOUT, "30")));
142145
setMaxFormContentSize(Integer.valueOf(properties.getProperty(REQUEST_CONTENT_SIZE_KEY, String.valueOf(DEFAULT_REQUEST_CONTENT_SIZE))));
146+
setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
143147
} catch (final IOException e) {
144148
LOG.warn("Failed to read configuration from server.properties file", e);
145149
} finally {
@@ -191,6 +195,7 @@ public void start() throws Exception {
191195
// Extra config options
192196
server.setStopAtShutdown(true);
193197
server.setAttribute(ContextHandler.MAX_FORM_CONTENT_SIZE_KEY, maxFormContentSize);
198+
server.setAttribute(ContextHandler.MAX_FORM_KEYS_KEY, maxFormKeys);
194199

195200
// HTTPS Connector
196201
createHttpsConnector(httpConfig);
@@ -263,6 +268,7 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
263268
webApp.setContextPath(contextPath);
264269
webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
265270
webApp.setMaxFormContentSize(maxFormContentSize);
271+
webApp.setMaxFormKeys(maxFormKeys);
266272

267273
// GZIP handler
268274
final GzipHandler gzipHandler = new GzipHandler();
@@ -365,4 +371,8 @@ public void setSessionTimeout(int sessionTimeout) {
365371
public void setMaxFormContentSize(int maxFormContentSize) {
366372
this.maxFormContentSize = maxFormContentSize;
367373
}
374+
375+
public void setMaxFormKeys(int maxFormKeys) {
376+
this.maxFormKeys = maxFormKeys;
377+
}
368378
}

‎engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
5353
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
5454
import org.apache.cloudstack.utils.identity.ManagementServerNode;
55+
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
5556
import org.apache.commons.lang3.BooleanUtils;
5657
import org.apache.log4j.Logger;
5758
import org.apache.log4j.MDC;
@@ -569,27 +570,27 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi
569570
}
570571
for (int i = 0; i < cmd.length; i++) {
571572
try {
573+
if (s_logger.isDebugEnabled()) {
574+
s_logger.debug("process connection to issue " + ReflectionToStringBuilderUtils.reflectCollection(cmd[i]) + " forRebalance == " + forRebalance);
575+
}
572576
monitor.second().processConnect(host, cmd[i], forRebalance);
573-
} catch (final Exception e) {
574-
if (e instanceof ConnectionException) {
575-
final ConnectionException ce = (ConnectionException)e;
576-
if (ce.isSetupError()) {
577-
s_logger.warn("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage());
578-
handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
579-
throw ce;
580-
} else {
581-
s_logger.info("Monitor " + monitor.second().getClass().getSimpleName() + " says not to continue the connect process for " + hostId + " due to " + e.getMessage());
582-
handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
583-
return attache;
584-
}
585-
} else if (e instanceof HypervisorVersionChangedException) {
586-
handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
587-
throw new CloudRuntimeException("Unable to connect " + attache.getId(), e);
588-
} else {
589-
s_logger.error("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage(), e);
577+
} catch (final ConnectionException ce) {
578+
if (ce.isSetupError()) {
579+
s_logger.warn("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + ce.getMessage());
590580
handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
591-
throw new CloudRuntimeException("Unable to connect " + attache.getId(), e);
581+
throw ce;
582+
} else {
583+
s_logger.info("Monitor " + monitor.second().getClass().getSimpleName() + " says not to continue the connect process for " + hostId + " due to " + ce.getMessage());
584+
handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
585+
return attache;
592586
}
587+
} catch (final HypervisorVersionChangedException hvce) {
588+
handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
589+
throw new CloudRuntimeException("Unable to connect " + (attache == null ? "<unknown agent>" : attache.getId()), hvce);
590+
} catch (final Exception e) {
591+
s_logger.error("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage(), e);
592+
handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
593+
throw new CloudRuntimeException("Unable to connect " + (attache == null ? "<unknown agent>" : attache.getId()), e);
593594
}
594595
}
595596
}

‎engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -794,12 +794,16 @@ public void doInTransactionWithoutResult(final TransactionStatus status) {
794794
Long templateId = getRegisteredTemplateId(hypervisorAndTemplateName);
795795
if (templateId != null) {
796796
VMTemplateVO templateVO = vmTemplateDao.findById(templateId);
797-
TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(templateId, DataStoreRole.Image);
798-
String installPath = templateDataStoreVO.getInstallPath();
799-
if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) {
800-
continue;
801-
} else if (templateVO != null) {
797+
TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId);
798+
if (templateDataStoreVO != null) {
799+
String installPath = templateDataStoreVO.getInstallPath();
800+
if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) {
801+
continue;
802+
}
803+
}
804+
if (templateVO != null) {
802805
registerTemplate(hypervisorAndTemplateName, storeUrlAndId, templateVO, templateDataStoreVO, filePath);
806+
updateRegisteredTemplateDetails(templateId, hypervisorAndTemplateName);
803807
continue;
804808
}
805809
}
@@ -823,6 +827,11 @@ public void doInTransactionWithoutResult(final TransactionStatus status) {
823827
}
824828

825829
private void updateRegisteredTemplateDetails(Long templateId, Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) {
830+
Pair<Hypervisor.HypervisorType, String> entry = new Pair<>(hypervisorAndTemplateName.getKey(), hypervisorAndTemplateName.getValue());
831+
updateRegisteredTemplateDetails(templateId, entry);
832+
}
833+
834+
private void updateRegisteredTemplateDetails(Long templateId, Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) {
826835
VMTemplateVO templateVO = vmTemplateDao.findById(templateId);
827836
templateVO.setTemplateType(Storage.TemplateType.SYSTEM);
828837
boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO);
@@ -832,11 +841,11 @@ private void updateRegisteredTemplateDetails(Long templateId, Map.Entry<Hypervis
832841
throw new CloudRuntimeException(errMsg);
833842
}
834843

835-
updateSystemVMEntries(templateId, hypervisorAndTemplateName.getKey());
844+
updateSystemVMEntries(templateId, hypervisorAndTemplateName.first());
836845

837846
// Change value of global configuration parameter router.template.* for the corresponding hypervisor and minreq.sysvmtemplate.version for the ACS version
838847
Map<String, String> configParams = new HashMap<>();
839-
configParams.put(RouterTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()), hypervisorAndTemplateName.getValue());
848+
configParams.put(RouterTemplateConfigurationNames.get(hypervisorAndTemplateName.first()), hypervisorAndTemplateName.second());
840849
configParams.put("minreq.sysvmtemplate.version", getSystemVmTemplateVersion());
841850
updateConfigurationParams(configParams);
842851
}

‎engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDao.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,13 @@ public interface NetworkPermissionDao extends GenericDao<NetworkPermissionVO, Lo
4040
*/
4141
void removeAllPermissions(long networkId);
4242

43+
/**
44+
* Removes all network permissions associated with a given account.
45+
*
46+
* @param accountId The ID of the account from which all network permissions will be removed.
47+
*/
48+
void removeAccountPermissions(long accountId);
49+
4350
/**
4451
* Find a Network permission by networkId, accountName, and domainId
4552
*

‎engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDaoImpl.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public class NetworkPermissionDaoImpl extends GenericDaoBase<NetworkPermissionVO
3535

3636
private SearchBuilder<NetworkPermissionVO> NetworkAndAccountSearch;
3737
private SearchBuilder<NetworkPermissionVO> NetworkIdSearch;
38+
private SearchBuilder<NetworkPermissionVO> accountSearch;
3839
private GenericSearchBuilder<NetworkPermissionVO, Long> FindNetworkIdsByAccount;
3940

4041
protected NetworkPermissionDaoImpl() {
@@ -47,6 +48,10 @@ protected NetworkPermissionDaoImpl() {
4748
NetworkIdSearch.and("networkId", NetworkIdSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
4849
NetworkIdSearch.done();
4950

51+
accountSearch = createSearchBuilder();
52+
accountSearch.and("accountId", accountSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
53+
accountSearch.done();
54+
5055
FindNetworkIdsByAccount = createSearchBuilder(Long.class);
5156
FindNetworkIdsByAccount.select(null, SearchCriteria.Func.DISTINCT, FindNetworkIdsByAccount.entity().getNetworkId());
5257
FindNetworkIdsByAccount.and("account", FindNetworkIdsByAccount.entity().getAccountId(), SearchCriteria.Op.IN);
@@ -71,6 +76,16 @@ public void removeAllPermissions(long networkId) {
7176
expunge(sc);
7277
}
7378

79+
@Override
80+
public void removeAccountPermissions(long accountId) {
81+
SearchCriteria<NetworkPermissionVO> sc = accountSearch.create();
82+
sc.setParameters("accountId", accountId);
83+
int networkPermissionRemoved = expunge(sc);
84+
if (networkPermissionRemoved > 0) {
85+
s_logger.debug(String.format("Removed [%s] network permission(s) for the account with Id [%s]", networkPermissionRemoved, accountId));
86+
}
87+
}
88+
7489
@Override
7590
public NetworkPermissionVO findByNetworkAndAccount(long networkId, long accountId) {
7691
SearchCriteria<NetworkPermissionVO> sc = NetworkAndAccountSearch.create();

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
431431

432432
protected static final String LOCAL_STORAGE_PATH = "local.storage.path";
433433
protected static final String LOCAL_STORAGE_UUID = "local.storage.uuid";
434-
protected static final String DEFAULT_LOCAL_STORAGE_PATH = "/var/lib/libvirt/images/";
434+
public static final String DEFAULT_LOCAL_STORAGE_PATH = "/var/lib/libvirt/images";
435435

436436
protected List<String> localStoragePaths = new ArrayList<>();
437437
protected List<String> localStorageUUIDs = new ArrayList<>();
@@ -2646,7 +2646,7 @@ protected DevicesDef createDevicesDef(VirtualMachineTO vmTO, GuestDef guest, int
26462646
Map<String, String> details = vmTO.getDetails();
26472647

26482648
boolean isIothreadsEnabled = details != null && details.containsKey(VmDetailConstants.IOTHREADS);
2649-
devices.addDevice(createSCSIDef(vcpus, isIothreadsEnabled));
2649+
addSCSIControllers(devices, vcpus, vmTO.getDisks().length, isIothreadsEnabled);
26502650
}
26512651
return devices;
26522652
}
@@ -2684,8 +2684,19 @@ protected ChannelDef createChannelDef(VirtualMachineTO vmTO) {
26842684
* Creates Virtio SCSI controller. <br>
26852685
* The respective Virtio SCSI XML definition is generated only if the VM's Disk Bus is of ISCSI.
26862686
*/
2687-
protected SCSIDef createSCSIDef(int vcpus, boolean isIothreadsEnabled) {
2688-
return new SCSIDef((short)0, 0, 0, 9, 0, vcpus, isIothreadsEnabled);
2687+
protected SCSIDef createSCSIDef(short index, int vcpus, boolean isIothreadsEnabled) {
2688+
return new SCSIDef(index, 0, 0, 9 + index, 0, vcpus, isIothreadsEnabled);
2689+
}
2690+
2691+
2692+
private void addSCSIControllers(DevicesDef devices, int vcpus, int diskCount, boolean isIothreadsEnabled) {
2693+
int controllers = diskCount / 7;
2694+
if (diskCount % 7 != 0) {
2695+
controllers++;
2696+
}
2697+
for (int i = 0; i < controllers; i++) {
2698+
devices.addDevice(createSCSIDef((short)i, vcpus, isIothreadsEnabled));
2699+
}
26892700
}
26902701

26912702
protected ConsoleDef createConsoleDef() {

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteStoragePoolCommandWrapper.java‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,28 +22,78 @@
2222
import com.cloud.agent.api.Answer;
2323
import com.cloud.agent.api.DeleteStoragePoolCommand;
2424
import com.cloud.agent.api.to.StorageFilerTO;
25+
import com.cloud.agent.dao.impl.PropertiesStorage;
26+
import com.cloud.agent.properties.AgentProperties;
27+
import com.cloud.agent.properties.AgentPropertiesFileHandler;
2528
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
2629
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
2730
import com.cloud.resource.CommandWrapper;
2831
import com.cloud.resource.ResourceWrapper;
32+
import com.cloud.storage.Storage;
2933
import com.cloud.utils.exception.CloudRuntimeException;
3034

35+
import java.util.Arrays;
36+
import java.util.HashMap;
37+
import java.util.stream.Collectors;
38+
3139
@ResourceWrapper(handles = DeleteStoragePoolCommand.class)
3240
public final class LibvirtDeleteStoragePoolCommandWrapper extends CommandWrapper<DeleteStoragePoolCommand, Answer, LibvirtComputingResource> {
3341
@Override
3442
public Answer execute(final DeleteStoragePoolCommand command, final LibvirtComputingResource libvirtComputingResource) {
3543
try {
3644
// if getRemoveDatastore() is true, then we are dealing with managed storage and can skip the delete logic here
3745
if (!command.getRemoveDatastore()) {
38-
final StorageFilerTO pool = command.getPool();
39-
final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
40-
41-
storagePoolMgr.deleteStoragePool(pool.getType(), pool.getUuid());
46+
handleStoragePoolDeletion(command, libvirtComputingResource);
4247
}
43-
4448
return new Answer(command);
4549
} catch (final CloudRuntimeException e) {
4650
return new Answer(command, false, e.toString());
4751
}
4852
}
53+
54+
private void handleStoragePoolDeletion(final DeleteStoragePoolCommand command, final LibvirtComputingResource libvirtComputingResource) {
55+
final StorageFilerTO pool = command.getPool();
56+
final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
57+
storagePoolMgr.deleteStoragePool(pool.getType(), pool.getUuid());
58+
59+
if (isLocalStorageAndNotHavingDefaultPath(pool, libvirtComputingResource)) {
60+
updateLocalStorageProperties(pool);
61+
}
62+
}
63+
64+
private boolean isLocalStorageAndNotHavingDefaultPath(final StorageFilerTO pool, final LibvirtComputingResource libvirtComputingResource) {
65+
return Storage.StoragePoolType.Filesystem.equals(pool.getType())
66+
&& !libvirtComputingResource.DEFAULT_LOCAL_STORAGE_PATH.equals(pool.getPath());
67+
}
68+
69+
private void updateLocalStorageProperties(final StorageFilerTO pool) {
70+
String localStoragePath = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.LOCAL_STORAGE_PATH);
71+
String localStorageUuid = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.LOCAL_STORAGE_UUID);
72+
73+
String uuidToRemove = pool.getUuid();
74+
String pathToRemove = pool.getPath();
75+
76+
if (localStorageUuid != null && uuidToRemove != null) {
77+
localStorageUuid = Arrays.stream(localStorageUuid.split(","))
78+
.filter(uuid -> !uuid.equals(uuidToRemove))
79+
.collect(Collectors.joining(","));
80+
}
81+
82+
if (localStoragePath != null && pathToRemove != null) {
83+
localStoragePath = Arrays.stream(localStoragePath.split(","))
84+
.filter(path -> !path.equals(pathToRemove))
85+
.collect(Collectors.joining(","));
86+
}
87+
88+
PropertiesStorage agentProperties = new PropertiesStorage();
89+
agentProperties.configure("AgentProperties", new HashMap<String, Object>());
90+
91+
if (localStorageUuid != null) {
92+
agentProperties.persist(AgentProperties.LOCAL_STORAGE_UUID.getName(), localStorageUuid);
93+
}
94+
95+
if (localStoragePath != null) {
96+
agentProperties.persist(AgentProperties.LOCAL_STORAGE_PATH.getName(), localStoragePath);
97+
}
98+
}
4999
}

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,9 @@ public void testCreateDevicesWithSCSIDisk() {
453453
to.setDetails(new HashMap<>());
454454
to.setPlatformEmulator("Other PV Virtio-SCSI");
455455

456+
final DiskTO diskTO = Mockito.mock(DiskTO.class);
457+
to.setDisks(new DiskTO[]{diskTO});
458+
456459
GuestDef guest = new GuestDef();
457460
guest.setGuestType(GuestType.KVM);
458461

@@ -640,7 +643,7 @@ public void testCreateChannelDef() {
640643
public void testCreateSCSIDef() {
641644
VirtualMachineTO to = createDefaultVM(false);
642645

643-
SCSIDef scsiDef = libvirtComputingResourceSpy.createSCSIDef(to.getCpus(), false);
646+
SCSIDef scsiDef = libvirtComputingResourceSpy.createSCSIDef((short)0, to.getCpus(), false);
644647
Document domainDoc = parse(scsiDef.toString());
645648
verifyScsi(to, domainDoc, "");
646649
}

0 commit comments

Comments
 (0)