Skip to content

Commit 9434391

Browse files
shwstpprshwstppr
committed
Merge remote-tracking branch 'apache/4.20' into deployvm-improvements
2 parents 707b0ea + 7e2aa0e commit 9434391

File tree

69 files changed

+1334
-1053
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+1334
-1053
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ jobs:
236236
237237
- name: Install Python dependencies
238238
run: |
239-
python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid pynose
239+
python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose
240240
241241
- name: Install jacoco dependencies
242242
run: |

‎api/src/main/java/org/apache/cloudstack/api/ApiConstants.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -478,12 +478,11 @@ public class ApiConstants {
478478
public static final String STATE = "state";
479479
public static final String STATS = "stats";
480480
public static final String STATUS = "status";
481+
public static final String STORAGE_TYPE = "storagetype";
482+
public static final String STORAGE_POLICY = "storagepolicy";
483+
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
481484
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
482485
public static final String STORAGE_CUSTOM_STATS = "storagecustomstats";
483-
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
484-
public static final String STORAGE_POLICY = "storagepolicy";
485-
public static final String STORAGE_POOL = "storagepool";
486-
public static final String STORAGE_TYPE = "storagetype";
487486
public static final String SUBNET = "subnet";
488487
public static final String OWNER = "owner";
489488
public static final String SWAP_OWNER = "swapowner";

‎api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ public class HostResponse extends BaseResponseWithAnnotations {
152152
@Deprecated
153153
@SerializedName("memoryallocated")
154154
@Param(description = "the amount of the host's memory currently allocated")
155-
private Long memoryAllocated;
155+
private long memoryAllocated;
156156

157157
@SerializedName("memoryallocatedpercentage")
158158
@Param(description = "the amount of the host's memory currently allocated in percentage")
@@ -407,7 +407,7 @@ public void setMemWithOverprovisioning(String memWithOverprovisioning){
407407
this.memWithOverprovisioning=memWithOverprovisioning;
408408
}
409409

410-
public void setMemoryAllocated(Long memoryAllocated) {
410+
public void setMemoryAllocated(long memoryAllocated) {
411411
this.memoryAllocated = memoryAllocated;
412412
}
413413

@@ -687,8 +687,8 @@ public Long getMemoryTotal() {
687687
return memoryTotal;
688688
}
689689

690-
public Long getMemoryAllocated() {
691-
return memoryAllocated == null ? 0 : memoryAllocated;
690+
public long getMemoryAllocated() {
691+
return memoryAllocated;
692692
}
693693

694694
public void setMemoryAllocatedPercentage(String memoryAllocatedPercentage) {

‎engine/components-api/src/main/java/com/cloud/agent/AgentManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
* AgentManager manages hosts. It directly coordinates between the DAOs and the connections it manages.
3838
*/
3939
public interface AgentManager {
40-
static final ConfigKey<Integer> Wait = new ConfigKey<Integer>("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return",
40+
ConfigKey<Integer> Wait = new ConfigKey<Integer>("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return",
4141
true);
4242
ConfigKey<Boolean> EnableKVMAutoEnableDisable = new ConfigKey<>(Boolean.class,
4343
"enable.kvm.host.auto.enable.disable",
@@ -54,7 +54,7 @@ public interface AgentManager {
5454
"This timeout overrides the wait global config. This holds a comma separated key value pairs containing timeout (in seconds) for specific commands. " +
5555
"For example: DhcpEntryCommand=600, SavePasswordCommand=300, VmDataCommand=300", false);
5656

57-
public enum TapAgentsAction {
57+
enum TapAgentsAction {
5858
Add, Del, Contains,
5959
}
6060

‎engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,16 @@
2525
import javax.annotation.PostConstruct;
2626
import javax.inject.Inject;
2727

28+
import com.cloud.network.Network;
29+
import org.apache.commons.collections.MapUtils;
30+
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
31+
2832
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
2933
import org.apache.cloudstack.framework.events.Event;
3034
import org.apache.cloudstack.framework.events.EventBus;
3135
import org.apache.cloudstack.framework.events.EventDistributor;
32-
import org.apache.commons.collections.MapUtils;
3336
import org.apache.logging.log4j.LogManager;
3437
import org.apache.logging.log4j.Logger;
35-
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
3638

3739
import com.cloud.dc.DataCenterVO;
3840
import com.cloud.dc.dao.DataCenterDao;
@@ -243,4 +245,22 @@ private static void publishUsageEvent(String usageEventType, Long accountId, Lon
243245

244246
static final String Name = "management-server";
245247

248+
public static void publishNetworkCreation(Network network) {
249+
publishUsageEvent(EventTypes.EVENT_NETWORK_CREATE, network.getAccountId(), network.getDataCenterId(),
250+
network.getId(), network.getName(), network.getNetworkOfferingId(), null, null, null, network.getState().name(),
251+
network.getUuid());
252+
}
253+
254+
public static void publishNetworkUpdate(Network network) {
255+
publishUsageEvent(EventTypes.EVENT_NETWORK_UPDATE, network.getAccountId(), network.getDataCenterId(),
256+
network.getId(), network.getName(), network.getNetworkOfferingId(), null, network.getState().name(),
257+
Network.class.getName(), network.getUuid(), true);
258+
}
259+
260+
public static void publishNetworkDeletion(Network network) {
261+
publishUsageEvent(EventTypes.EVENT_NETWORK_DELETE, network.getAccountId(), network.getDataCenterId(),
262+
network.getId(), network.getName(), network.getNetworkOfferingId(), null, null, null,
263+
Network.class.getName(), network.getUuid());
264+
}
265+
246266
}

‎engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@
5151
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
5252
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
5353
import org.apache.cloudstack.utils.identity.ManagementServerNode;
54-
import org.apache.commons.collections.MapUtils;
5554
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
55+
import org.apache.commons.collections.MapUtils;
5656
import org.apache.commons.lang3.BooleanUtils;
5757
import org.apache.commons.lang3.StringUtils;
5858
import org.apache.logging.log4j.ThreadContext;
@@ -210,6 +210,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
210210
"Number of maximum concurrent new connections server allows for remote agents. " +
211211
"If set to zero (default value) then no limit will be enforced on concurrent new connections",
212212
false);
213+
protected final ConfigKey<Integer> RemoteAgentNewConnectionsMonitorInterval = new ConfigKey<>("Advanced", Integer.class, "agent.connections.monitor.interval", "1800",
214+
"Time in seconds to monitor the new agent connections and cleanup the expired connections.", false);
213215
protected final ConfigKey<Integer> AlertWait = new ConfigKey<Integer>("Advanced", Integer.class, "alert.wait", "1800",
214216
"Seconds to wait before alerting on a disconnected agent", true);
215217
protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<Integer>("Advanced", Integer.class, "direct.agent.load.size", "16",
@@ -726,9 +728,9 @@ public boolean start() {
726728

727729
_monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS);
728730

729-
final int cleanupTime = Wait.value();
730-
newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTime,
731-
cleanupTime, TimeUnit.MINUTES);
731+
final int agentConnectionsMonitorTimeInSecs = RemoteAgentNewConnectionsMonitorInterval.value();
732+
newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), agentConnectionsMonitorTimeInSecs,
733+
agentConnectionsMonitorTimeInSecs, TimeUnit.SECONDS);
732734

733735
return true;
734736
}
@@ -1857,27 +1859,21 @@ protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable {
18571859
@Override
18581860
protected void runInContext() {
18591861
logger.trace("Agent New Connections Monitor is started.");
1860-
final int cleanupTime = Wait.value();
1862+
final int cleanupTime = RemoteAgentNewConnectionsMonitorInterval.value();
18611863
Set<Map.Entry<String, Long>> entrySet = newAgentConnections.entrySet();
1862-
long cutOff = System.currentTimeMillis() - (cleanupTime * 60 * 1000L);
1863-
if (logger.isDebugEnabled()) {
1864-
List<String> expiredConnections = newAgentConnections.entrySet()
1865-
.stream()
1866-
.filter(e -> e.getValue() <= cutOff)
1867-
.map(Map.Entry::getKey)
1868-
.collect(Collectors.toList());
1869-
logger.debug(String.format("Currently %d active new connections, of which %d have expired - %s",
1870-
entrySet.size(),
1871-
expiredConnections.size(),
1872-
StringUtils.join(expiredConnections)));
1873-
}
1874-
for (Map.Entry<String, Long> entry : entrySet) {
1875-
if (entry.getValue() <= cutOff) {
1876-
if (logger.isTraceEnabled()) {
1877-
logger.trace(String.format("Cleaning up new agent connection for %s", entry.getKey()));
1878-
}
1879-
newAgentConnections.remove(entry.getKey());
1880-
}
1864+
long cutOff = System.currentTimeMillis() - (cleanupTime * 1000L);
1865+
List<String> expiredConnections = newAgentConnections.entrySet()
1866+
.stream()
1867+
.filter(e -> e.getValue() <= cutOff)
1868+
.map(Map.Entry::getKey)
1869+
.collect(Collectors.toList());
1870+
logger.debug("Currently {} active new connections, of which {} have expired - {}",
1871+
entrySet.size(),
1872+
expiredConnections.size(),
1873+
StringUtils.join(expiredConnections));
1874+
for (String connection : expiredConnections) {
1875+
logger.trace("Cleaning up new agent connection for {}", connection);
1876+
newAgentConnections.remove(connection);
18811877
}
18821878
}
18831879
}
@@ -1958,7 +1954,8 @@ public String getConfigComponentName() {
19581954
public ConfigKey<?>[] getConfigKeys() {
19591955
return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
19601956
DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
1961-
GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections };
1957+
GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections,
1958+
RemoteAgentNewConnectionsMonitorInterval };
19621959
}
19631960

19641961
protected class SetHostParamsListener implements Listener {

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
427427
static final ConfigKey<Long> VmOpCleanupInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cleanup.interval", "86400",
428428
"Interval to run the thread that cleans up the vm operations (in seconds)", false);
429429
static final ConfigKey<Long> VmOpCleanupWait = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cleanup.wait", "3600",
430-
"Time (in seconds) to wait before cleanuping up any vm work items", true);
430+
"Time (in seconds) to wait before cleaning up any vm work items", true);
431431
static final ConfigKey<Long> VmOpCancelInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cancel.interval", "3600",
432432
"Time (in seconds) to wait before cancelling a operation", false);
433433
static final ConfigKey<Boolean> VmDestroyForcestop = new ConfigKey<Boolean>("Advanced", Boolean.class, "vm.destroy.forcestop", "false",

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) {
595595
if (_networkOfferingDao.findByUniqueName(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworks) == null) {
596596
offering = _configMgr.createNetworkOffering(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworks,
597597
"Offering for Isolated VPC networks with Source Nat service enabled", TrafficType.Guest, null, false, Availability.Optional, null,
598-
defaultVPCOffProviders, true, Network.GuestType.Isolated, false, null, false, null, false, false, null, false, null, true, true, false, false, null, null, null,true, null, null, false);
598+
defaultVPCOffProviders, true, Network.GuestType.Isolated, false, null, true, null, false, false, null, false, null, true, true, false, false, null, null, null,true, null, null, false);
599599
}
600600

601601
//#6 - default vpc offering with no LB service
@@ -1528,8 +1528,6 @@ public Pair<NetworkGuru, NetworkVO> implementNetwork(final long networkId, final
15281528
if (isNetworkImplemented(network)) {
15291529
logger.debug("Network {} is already implemented", network);
15301530
implemented.set(guru, network);
1531-
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_UPDATE, network.getAccountId(), network.getDataCenterId(), network.getId(),
1532-
network.getName(), network.getNetworkOfferingId(), null, network.getState().name(), Network.class.getName(), network.getUuid(), true);
15331531
return implemented;
15341532
}
15351533

@@ -1585,9 +1583,8 @@ public Pair<NetworkGuru, NetworkVO> implementNetwork(final long networkId, final
15851583

15861584
network.setRestartRequired(false);
15871585
_networksDao.update(network.getId(), network);
1586+
UsageEventUtils.publishNetworkUpdate(network);
15881587
implemented.set(guru, network);
1589-
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_CREATE, network.getAccountId(), network.getDataCenterId(), network.getId(),
1590-
network.getName(), network.getNetworkOfferingId(), null, null, null, network.getState().name(), network.getUuid());
15911588
return implemented;
15921589
} catch (final NoTransitionException e) {
15931590
logger.error(e.getMessage());
@@ -3087,6 +3084,7 @@ public Network doInTransaction(final TransactionStatus status) {
30873084
if (updateResourceCount) {
30883085
_resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.network, isDisplayNetworkEnabled);
30893086
}
3087+
UsageEventUtils.publishNetworkCreation(network);
30903088

30913089
return network;
30923090
}
@@ -3168,13 +3166,14 @@ public boolean shutdownNetwork(final long networkId, final ReservationContext co
31683166
}
31693167
logger.debug("Lock is acquired for network {} as a part of network shutdown", network);
31703168

3171-
if (network.getState() == Network.State.Allocated) {
3172-
logger.debug("Network is already shutdown: {}", network);
3169+
final Network.State initialState = network.getState();
3170+
if (initialState == Network.State.Allocated) {
3171+
logger.debug(String.format("Network [%s] is in Allocated state, no need to shutdown.", network));
31733172
return true;
31743173
}
31753174

3176-
if (network.getState() != Network.State.Implemented && network.getState() != Network.State.Shutdown) {
3177-
logger.debug("Network is not implemented: {}", network);
3175+
if (initialState != Network.State.Implemented && initialState != Network.State.Shutdown) {
3176+
logger.debug("Network is not implemented: " + network);
31783177
return false;
31793178
}
31803179

@@ -3218,6 +3217,9 @@ public Boolean doInTransaction(final TransactionStatus status) {
32183217
}
32193218
_networksDao.update(networkFinal.getId(), networkFinal);
32203219
_networksDao.clearCheckForGc(networkId);
3220+
if (initialState == Network.State.Implemented) {
3221+
UsageEventUtils.publishNetworkUpdate(networkFinal);
3222+
}
32213223
result = true;
32223224
} else {
32233225
try {
@@ -3469,8 +3471,7 @@ public List<VlanVO> doInTransaction(TransactionStatus status) {
34693471
final Pair<Class<?>, Long> networkMsg = new Pair<Class<?>, Long>(Network.class, networkFinal.getId());
34703472
_messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, networkMsg);
34713473
}
3472-
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_DELETE, network.getAccountId(), network.getDataCenterId(), network.getId(),
3473-
network.getName(), network.getNetworkOfferingId(), null, null, null, Network.class.getName(), network.getUuid());
3474+
UsageEventUtils.publishNetworkDeletion(network);
34743475
return true;
34753476
} catch (final CloudRuntimeException e) {
34763477
logger.error("Failed to delete network", e);

‎engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -891,7 +891,7 @@ public void testShutdownNetworkInImplementingState() {
891891
boolean shutdownNetworkStatus = testOrchestrator.shutdownNetwork(networkId, reservationContext, false);
892892
Assert.assertFalse(shutdownNetworkStatus);
893893

894-
verify(network, times(3)).getState();
894+
verify(network).getState();
895895
verify(testOrchestrator._networksDao, times(1)).acquireInLockTable(networkId, NetworkLockTimeout.value());
896896
verify(testOrchestrator._networksDao, times(1)).releaseFromLockTable(networkId);
897897
}

‎engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'va
7676
CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
7777
CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
7878

79+
-- Re-apply VPC: update default network offering for vpc tier to conserve_mode=1 (#8309)
80+
UPDATE `cloud`.`network_offerings` SET conserve_mode=1 WHERE name='DefaultIsolatedNetworkOfferingForVpcNetworks';
81+
7982
-- Add featured column for guest_os_category
8083
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os_category', 'featured', 'tinyint(1) NOT NULL DEFAULT 0 COMMENT "whether the category is featured or not" AFTER `uuid`');
8184
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os_category', 'sort_key', 'int NOT NULL DEFAULT 0 COMMENT "sort key used for customising sort method" AFTER `featured`');

0 commit comments

Comments
 (0)