Merge branch '4.19' into healthcheck-4.19

Author: DaanHoogland

api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmd.java

@@ -19,6 +19,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.acl.RoleType;
@@ -103,14 +105,13 @@ public String getProtocol() {
 
     @Override
     public List<String> getSourceCidrList() {
-        if (cidrlist != null) {
+        if (CollectionUtils.isNotEmpty(cidrlist) && !(cidrlist.size() == 1 && StringUtils.isBlank(cidrlist.get(0)))) {
             return cidrlist;
         } else {
-            List<String> oneCidrList = new ArrayList<String>();
+            List<String> oneCidrList = new ArrayList<>();
             oneCidrList.add(NetUtils.ALL_IP4_CIDRS);
             return oneCidrList;
         }
-
     }
 
     // ///////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java

@@ -187,6 +187,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement
     @Param(description = "true network requires restart")
     private Boolean restartRequired;
 
+    @SerializedName(ApiConstants.SPECIFY_VLAN)
+    @Param(description = "true if network supports specifying vlan, false otherwise")
+    private Boolean specifyVlan;
+
     @SerializedName(ApiConstants.SPECIFY_IP_RANGES)
     @Param(description = "true if network supports specifying ip ranges, false otherwise")
     private Boolean specifyIpRanges;
@@ -487,6 +491,10 @@ public void setRestartRequired(Boolean restartRequired) {
         this.restartRequired = restartRequired;
     }
 
+    public void setSpecifyVlan(Boolean specifyVlan) {
+        this.specifyVlan = specifyVlan;
+    }
+
     public void setSpecifyIpRanges(Boolean specifyIpRanges) {
         this.specifyIpRanges = specifyIpRanges;
     }

api/src/test/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmdTest.java

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.firewall;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.cloud.utils.net.NetUtils;
+
+@RunWith(MockitoJUnitRunner.class)
+public class CreateFirewallRuleCmdTest {
+
+    private void validateAllIp4Cidr(final CreateFirewallRuleCmd cmd) {
+        Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
+        Assert.assertEquals(1, cmd.getSourceCidrList().size());
+        Assert.assertEquals(NetUtils.ALL_IP4_CIDRS, cmd.getSourceCidrList().get(0));
+    }
+
+    @Test
+    public void testGetSourceCidrList_Null() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        ReflectionTestUtils.setField(cmd, "cidrlist", null);
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_Empty() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        ReflectionTestUtils.setField(cmd, "cidrlist", new ArrayList<>());
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_NullFirstElement() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        List<String> list = new ArrayList<>();
+        list.add(null);
+        ReflectionTestUtils.setField(cmd, "cidrlist", list);
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_EmptyFirstElement() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        ReflectionTestUtils.setField(cmd, "cidrlist", Collections.singletonList("  "));
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_Valid() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        String cidr = "10.1.1.1/22";
+        ReflectionTestUtils.setField(cmd, "cidrlist", Collections.singletonList(cidr));
+        Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
+        Assert.assertEquals(1, cmd.getSourceCidrList().size());
+        Assert.assertEquals(cidr, cmd.getSourceCidrList().get(0));
+    }
+
+    @Test
+    public void testGetSourceCidrList_EmptyFirstElementButMore() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        String cidr = "10.1.1.1/22";
+        ReflectionTestUtils.setField(cmd, "cidrlist", Arrays.asList("  ", cidr));
+        Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
+        Assert.assertEquals(2, cmd.getSourceCidrList().size());
+        Assert.assertEquals(cidr, cmd.getSourceCidrList().get(1));
+    }
+}

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java

@@ -81,6 +81,9 @@ public interface NetworkOrchestrationService {
     ConfigKey<Integer> NetworkLockTimeout = new ConfigKey<Integer>(Integer.class, NetworkLockTimeoutCK, "Network", "600",
         "Lock wait timeout (seconds) while implementing network", true, Scope.Global, null);
 
+    ConfigKey<String> DeniedRoutes = new ConfigKey<String>(String.class, "denied.routes", "Network", "",
+            "Routes that are denied, can not be used for Static Routes creation for the VPC Private Gateway", true, ConfigKey.Scope.Zone, null);
+
     ConfigKey<String> GuestDomainSuffix = new ConfigKey<String>(String.class, GuestDomainSuffixCK, "Network", "cloud.internal",
         "Default domain name for vms inside virtualized networks fronted by router", true, ConfigKey.Scope.Zone, null);
 

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -4792,7 +4792,7 @@ public String getConfigComponentName() {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout,
+        return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes,
                 GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion,
                 PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled,
                 TUNGSTEN_ENABLED };

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -60,6 +60,7 @@
 import com.cloud.org.Managed;
 import com.cloud.resource.ResourceState;
 import com.cloud.utils.DateUtil;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.db.Attribute;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.Filter;
@@ -83,13 +84,13 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     private static final Logger status_logger = Logger.getLogger(Status.class);
     private static final Logger state_logger = Logger.getLogger(ResourceState.class);
 
-    private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
-                                                             + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered "
-                                                             + "WHERE tag IN(%s) AND is_tag_a_rule = 0 "
+    private static final String LIST_HOST_IDS_BY_HOST_TAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
+                                                             + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag,is_tag_a_rule) AS filtered "
+                                                             + "WHERE tag IN (%s) AND (is_tag_a_rule = 0 OR is_tag_a_rule IS NULL) "
                                                              + "GROUP BY host_id "
                                                              + "HAVING tag_count = %s ";
     private static final String SEPARATOR = ",";
-    private static final String LIST_CLUSTERID_FOR_HOST_TAG = "select distinct cluster_id from host join ( %s ) AS selected_hosts ON host.id = selected_hosts.host_id";
+    private static final String LIST_CLUSTER_IDS_FOR_HOST_TAGS = "select distinct cluster_id from host join ( %s ) AS selected_hosts ON host.id = selected_hosts.host_id";
     private static final String GET_HOSTS_OF_ACTIVE_VMS = "select h.id " +
             "from vm_instance vm " +
             "join host h on (vm.host_id=h.id) " +
@@ -785,6 +786,15 @@ public void markHostsAsDisconnected(long msId, long lastPing) {
 
     @Override
     public List<HostVO> listByHostTag(Host.Type type, Long clusterId, Long podId, long dcId, String hostTag) {
+        return listHostsWithOrWithoutHostTags(type, clusterId, podId, dcId, hostTag, true);
+    }
+
+    private List<HostVO> listHostsWithOrWithoutHostTags(Host.Type type, Long clusterId, Long podId, long dcId, String hostTags, boolean withHostTags) {
+        if (StringUtils.isEmpty(hostTags)) {
+            s_logger.debug("Host tags not specified, to list hosts");
+            return new ArrayList();
+        }
+
         SearchBuilder<HostVO> hostSearch = createSearchBuilder();
         HostVO entity = hostSearch.entity();
         hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ);
@@ -795,7 +805,9 @@ public List<HostVO> listByHostTag(Host.Type type, Long clusterId, Long podId, lo
         hostSearch.and("resourceState", entity.getResourceState(), SearchCriteria.Op.EQ);
 
         SearchCriteria<HostVO> sc = hostSearch.create();
-        sc.setParameters("type", type.toString());
+        if (type != null) {
+            sc.setParameters("type", type.toString());
+        }
         if (podId != null) {
             sc.setParameters("pod", podId);
         }
@@ -806,27 +818,38 @@ public List<HostVO> listByHostTag(Host.Type type, Long clusterId, Long podId, lo
         sc.setParameters("status", Status.Up.toString());
         sc.setParameters("resourceState", ResourceState.Enabled.toString());
 
-        List<HostVO> tmpHosts = listBy(sc);
-        List<HostVO> correctHostsByHostTags = new ArrayList();
-        List<Long> hostIdsByComputeOffTags = findHostByComputeOfferings(hostTag);
+        List<HostVO> upAndEnabledHosts = listBy(sc);
+        if (CollectionUtils.isEmpty(upAndEnabledHosts)) {
+            return new ArrayList();
+        }
 
-        tmpHosts.forEach((host) -> { if(hostIdsByComputeOffTags.contains(host.getId())) correctHostsByHostTags.add(host);});
+        List<Long> hostIdsByHostTags = findHostIdsByHostTags(hostTags);
+        if (CollectionUtils.isEmpty(hostIdsByHostTags)) {
+            return withHostTags ? new ArrayList() : upAndEnabledHosts;
+        }
 
-        return correctHostsByHostTags;
+        if (withHostTags) {
+            List<HostVO> upAndEnabledHostsWithHostTags = new ArrayList();
+            upAndEnabledHosts.forEach((host) -> { if (hostIdsByHostTags.contains(host.getId())) upAndEnabledHostsWithHostTags.add(host);});
+            return upAndEnabledHostsWithHostTags;
+        } else {
+            List<HostVO> upAndEnabledHostsWithoutHostTags = new ArrayList();
+            upAndEnabledHosts.forEach((host) -> { if (!hostIdsByHostTags.contains(host.getId())) upAndEnabledHostsWithoutHostTags.add(host);});
+            return upAndEnabledHostsWithoutHostTags;
+        }
     }
 
     @Override
     public List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) {
+        if (StringUtils.isNotEmpty(haTag)) {
+            return listHostsWithOrWithoutHostTags(type, clusterId, podId, dcId, haTag, false);
+        }
+
         SearchBuilder<HostTagVO> hostTagSearch = _hostTagsDao.createSearchBuilder();
         hostTagSearch.and();
         hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ);
         hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL);
         hostTagSearch.cp();
-        if (haTag != null && !haTag.isEmpty()) {
-            hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ);
-            hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
-            hostTagSearch.cp();
-        }
 
         SearchBuilder<HostVO> hostSearch = createSearchBuilder();
 
@@ -837,18 +860,12 @@ public List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Lon
         hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ);
         hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
 
-
         hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER);
 
-
         SearchCriteria<HostVO> sc = hostSearch.create();
 
         sc.setJoinParameters("hostTagSearch", "isTagARule", false);
 
-        if (haTag != null && !haTag.isEmpty()) {
-            sc.setJoinParameters("hostTagSearch", "tag", haTag);
-        }
-
         if (type != null) {
             sc.setParameters("type", type);
         }
@@ -1300,19 +1317,19 @@ public List<HostVO> listAllHostsThatHaveNoRuleTag(Type type, Long clusterId, Lon
     }
 
     @Override
-    public List<Long> listClustersByHostTag(String computeOfferingTags) {
+    public List<Long> listClustersByHostTag(String hostTags) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        String sql = this.LIST_CLUSTERID_FOR_HOST_TAG;
+        String selectStmtToListClusterIdsByHostTags = this.LIST_CLUSTER_IDS_FOR_HOST_TAGS;
         PreparedStatement pstmt = null;
         List<Long> result = new ArrayList();
-        List<String> tags = Arrays.asList(computeOfferingTags.split(this.SEPARATOR));
-        String subselect = getHostIdsByComputeTags(tags);
-        sql = String.format(sql, subselect);
+        List<String> tags = Arrays.asList(hostTags.split(this.SEPARATOR));
+        String selectStmtToListHostIdsByHostTags = getSelectStmtToListHostIdsByHostTags(tags);
+        selectStmtToListClusterIdsByHostTags = String.format(selectStmtToListClusterIdsByHostTags, selectStmtToListHostIdsByHostTags);
 
         try {
-            pstmt = txn.prepareStatement(sql);
+            pstmt = txn.prepareStatement(selectStmtToListClusterIdsByHostTags);
 
-            for(int i = 0; i < tags.size(); i++){
+            for (int i = 0; i < tags.size(); i++){
                 pstmt.setString(i+1, tags.get(i));
             }
 
@@ -1323,20 +1340,20 @@ public List<Long> listClustersByHostTag(String computeOfferingTags) {
             pstmt.close();
             return result;
         } catch (SQLException e) {
-            throw new CloudRuntimeException("DB Exception on: " + sql, e);
+            throw new CloudRuntimeException("DB Exception on: " + selectStmtToListClusterIdsByHostTags, e);
         }
     }
 
-    private List<Long> findHostByComputeOfferings(String computeOfferingTags){
+    private List<Long> findHostIdsByHostTags(String hostTags){
         TransactionLegacy txn = TransactionLegacy.currentTxn();
         PreparedStatement pstmt = null;
         List<Long> result = new ArrayList();
-        List<String> tags = Arrays.asList(computeOfferingTags.split(this.SEPARATOR));
-        String select = getHostIdsByComputeTags(tags);
+        List<String> tags = Arrays.asList(hostTags.split(this.SEPARATOR));
+        String selectStmtToListHostIdsByHostTags = getSelectStmtToListHostIdsByHostTags(tags);
         try {
-            pstmt = txn.prepareStatement(select);
+            pstmt = txn.prepareStatement(selectStmtToListHostIdsByHostTags);
 
-            for(int i = 0; i < tags.size(); i++){
+            for (int i = 0; i < tags.size(); i++){
                 pstmt.setString(i+1, tags.get(i));
             }
 
@@ -1347,7 +1364,7 @@ private List<Long> findHostByComputeOfferings(String computeOfferingTags){
             pstmt.close();
             return result;
         } catch (SQLException e) {
-            throw new CloudRuntimeException("DB Exception on: " + select, e);
+            throw new CloudRuntimeException("DB Exception on: " + selectStmtToListHostIdsByHostTags, e);
         }
     }
 
@@ -1372,10 +1389,10 @@ public List<Long> findClustersThatMatchHostTagRule(String computeOfferingTags) {
         return new ArrayList<>(result);
     }
 
-    private String getHostIdsByComputeTags(List<String> offeringTags){
+    private String getSelectStmtToListHostIdsByHostTags(List<String> hostTags){
         List<String> questionMarks = new ArrayList();
-        offeringTags.forEach((tag) -> { questionMarks.add("?"); });
-        return String.format(this.LIST_HOST_IDS_BY_COMPUTETAGS, String.join(",", questionMarks),questionMarks.size());
+        hostTags.forEach((tag) -> { questionMarks.add("?"); });
+        return String.format(this.LIST_HOST_IDS_BY_HOST_TAGS, String.join(",", questionMarks), questionMarks.size());
     }
 
     @Override

engine/schema/src/main/java/com/cloud/user/UserAccountVO.java

@@ -35,6 +35,8 @@
 
 import com.cloud.utils.db.Encrypt;
 import com.cloud.utils.db.GenericDao;
+
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 import org.apache.commons.lang3.StringUtils;
 
 @Entity
@@ -368,4 +370,9 @@ public Map<String, String> getDetails() {
     public void setDetails(Map<String, String> details) {
         this.details = details;
     }
+
+    @Override
+    public String toString() {
+        return String.format("User %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "name", "uuid"));
+    }
 }