New feature: Implicit host tags (apache#8929)

* Merge two HostTagVO and HostTagDaoImpl

* Implicit host tags

* PR8929: add since

* Update variable names

* Update 8929: add unit test in LibvirtComputingResourceTest

* Update 8929: add explicithosttags in response

* Update 8929 UI: Update explicit host tags

* Update 8929: remove host tags and change labels on UI

* Update 8929: update host_view to use explicit_host_tags.is_tag_a_rule

* Update: ui polish for host tags

* Update 8929: fix UI error if no host tags

Author: weizhouapache

agent/conf/agent.properties

@@ -430,3 +430,6 @@ iscsi.session.cleanup.enabled=false
 # If set to "true", the agent will register for libvirt domain events, allowing for immediate updates on crashed or
 # unexpectedly stopped. Experimental, requires agent restart.
 # libvirt.events.enabled=false
+
+# Implicit host tags managed by agent.properties
+# host.tags=

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -811,6 +811,13 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<String> KEYSTORE_PASSPHRASE = new Property<>(KeyStoreUtils.KS_PASSPHRASE_PROPERTY, null, String.class);
 
+    /**
+     * Implicit host tags
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -266,6 +266,7 @@ public class ApiConstants {
     public static final String IS_EDGE = "isedge";
     public static final String IS_EXTRACTABLE = "isextractable";
     public static final String IS_FEATURED = "isfeatured";
+    public static final String IS_IMPLICIT = "isimplicit";
     public static final String IS_PORTABLE = "isportable";
     public static final String IS_PUBLIC = "ispublic";
     public static final String IS_PERSISTENT = "ispersistent";

api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java

@@ -208,6 +208,14 @@ public class HostForMigrationResponse extends BaseResponse {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName("explicithosttags")
+    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0")
+    private String explicitHostTags;
+
+    @SerializedName("implicithosttags")
+    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0")
+    private String implicitHostTags;
+
     @SerializedName("hasenoughcapacity")
     @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise")
     private Boolean hasEnoughCapacity;
@@ -414,6 +422,14 @@ public void setHostTags(String hostTags) {
         this.hostTags = hostTags;
     }
 
+    public void setExplicitHostTags(String explicitHostTags) {
+        this.explicitHostTags = explicitHostTags;
+    }
+
+    public void setImplicitHostTags(String implicitHostTags) {
+        this.implicitHostTags = implicitHostTags;
+    }
+
     public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
         this.hasEnoughCapacity = hasEnoughCapacity;
     }

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -221,6 +221,14 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName("explicithosttags")
+    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0")
+    private String explicitHostTags;
+
+    @SerializedName("implicithosttags")
+    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0")
+    private String implicitHostTags;
+
     @SerializedName(ApiConstants.IS_TAG_A_RULE)
     @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
     private Boolean isTagARule;
@@ -459,6 +467,22 @@ public void setHostTags(String hostTags) {
         this.hostTags = hostTags;
     }
 
+    public String getExplicitHostTags() {
+        return explicitHostTags;
+    }
+
+    public void setExplicitHostTags(String explicitHostTags) {
+        this.explicitHostTags = explicitHostTags;
+    }
+
+    public String getImplicitHostTags() {
+        return implicitHostTags;
+    }
+
+    public void setImplicitHostTags(String implicitHostTags) {
+        this.implicitHostTags = implicitHostTags;
+    }
+
     public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
         this.hasEnoughCapacity = hasEnoughCapacity;
     }

api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java

@@ -19,6 +19,7 @@
 import com.google.gson.annotations.SerializedName;
 import com.cloud.serializer.Param;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
 public class HostTagResponse extends BaseResponse {
@@ -34,6 +35,10 @@ public class HostTagResponse extends BaseResponse {
     @Param(description = "the name of the host tag")
     private String name;
 
+    @SerializedName(ApiConstants.IS_IMPLICIT)
+    @Param(description = "true if the host tag is implicit", since = "4.20.0")
+    private boolean isImplicit;
+
     public String getId() {
         return id;
     }
@@ -57,4 +62,12 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
+
+    public boolean isImplicit() {
+        return isImplicit;
+    }
+
+    public void setImplicit(boolean implicit) {
+        isImplicit = implicit;
+    }
 }

core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java

@@ -174,6 +174,10 @@ public void setHostTags(String hostTag) {
         this.hostTags.add(hostTag);
     }
 
+    public void setHostTags(List<String> hostTags) {
+        this.hostTags = hostTags;
+    }
+
     public  HashMap<String, HashMap<String, VgpuTypesInfo>> getGpuGroupDetails() {
         return groupDetails;
     }

engine/schema/src/main/java/com/cloud/host/HostTagVO.java

@@ -40,6 +40,9 @@ public class HostTagVO implements InternalIdentity {
     @Column(name = "tag")
     private String tag;
 
+    @Column(name = "is_implicit")
+    private boolean isImplicit = false;
+
     @Column(name = "is_tag_a_rule")
     private boolean isTagARule;
 
@@ -74,6 +77,13 @@ public boolean getIsTagARule() {
         return isTagARule;
     }
 
+    public void setIsImplicit(boolean isImplicit) {
+        this.isImplicit = isImplicit;
+    }
+
+    public boolean getIsImplicit() {
+        return isImplicit;
+    }
 
     @Override
     public long getId() {

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java

@@ -20,6 +20,7 @@
 
 import com.cloud.host.HostTagVO;
 import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.api.response.HostTagResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
@@ -35,6 +36,13 @@ public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
 
     void deleteTags(long hostId);
 
+    boolean updateImplicitTags(long hostId, List<String> hostTags);
+
+    List<HostTagVO> getExplicitHostTags(long hostId);
+
     List<HostTagVO> findHostRuleTags();
 
+    HostTagResponse newHostTagResponse(HostTagVO hostTag);
+
+    List<HostTagVO> searchByIds(Long... hostTagIds);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java

@@ -16,10 +16,14 @@
 // under the License.
 package com.cloud.host.dao;
 
+import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.cloudstack.api.response.HostTagResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
 import com.cloud.host.HostTagVO;
@@ -30,14 +34,23 @@
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.SearchCriteria.Func;
 
+import javax.inject.Inject;
+
 @Component
 public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements HostTagsDao, Configurable {
     protected final SearchBuilder<HostTagVO> HostSearch;
     protected final GenericSearchBuilder<HostTagVO, String> DistinctImplictTagsSearch;
+    private final SearchBuilder<HostTagVO> stSearch;
+    private final SearchBuilder<HostTagVO> tagIdsearch;
+    private final SearchBuilder<HostTagVO> ImplicitTagsSearch;
+
+    @Inject
+    private ConfigurationDao _configDao;
 
     public HostTagsDaoImpl() {
         HostSearch = createSearchBuilder();
         HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        HostSearch.and("isImplicit", HostSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
         HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ);
         HostSearch.done();
 
@@ -46,6 +59,19 @@ public HostTagsDaoImpl() {
         DistinctImplictTagsSearch.and("hostIds", DistinctImplictTagsSearch.entity().getHostId(), SearchCriteria.Op.IN);
         DistinctImplictTagsSearch.and("implicitTags", DistinctImplictTagsSearch.entity().getTag(), SearchCriteria.Op.IN);
         DistinctImplictTagsSearch.done();
+
+        stSearch = createSearchBuilder();
+        stSearch.and("idIN", stSearch.entity().getId(), SearchCriteria.Op.IN);
+        stSearch.done();
+
+        tagIdsearch = createSearchBuilder();
+        tagIdsearch.and("id", tagIdsearch.entity().getId(), SearchCriteria.Op.EQ);
+        tagIdsearch.done();
+
+        ImplicitTagsSearch = createSearchBuilder();
+        ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
+        ImplicitTagsSearch.done();
     }
 
     @Override
@@ -74,6 +100,36 @@ public void deleteTags(long hostId) {
         txn.commit();
     }
 
+    @Override
+    public boolean updateImplicitTags(long hostId, List<String> hostTags) {
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+        SearchCriteria<HostTagVO> sc = ImplicitTagsSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", true);
+        boolean expunged = expunge(sc) > 0;
+        boolean persisted = false;
+        for (String tag : hostTags) {
+            if (StringUtils.isNotBlank(tag)) {
+                HostTagVO vo = new HostTagVO(hostId, tag.trim());
+                vo.setIsImplicit(true);
+                persist(vo);
+                persisted = true;
+            }
+        }
+        txn.commit();
+        return expunged || persisted;
+    }
+
+    @Override
+    public List<HostTagVO> getExplicitHostTags(long hostId) {
+        SearchCriteria<HostTagVO> sc = ImplicitTagsSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", false);
+
+        return search(sc, null);
+    }
+
     @Override
     public List<HostTagVO> findHostRuleTags() {
         SearchCriteria<HostTagVO> sc = HostSearch.create();
@@ -89,6 +145,7 @@ public void persist(long hostId, List<String> hostTags, Boolean isTagARule) {
         txn.start();
         SearchCriteria<HostTagVO> sc = HostSearch.create();
         sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", false);
         expunge(sc);
 
         for (String tag : hostTags) {
@@ -110,4 +167,72 @@ public ConfigKey<?>[] getConfigKeys() {
     public String getConfigComponentName() {
         return HostTagsDaoImpl.class.getSimpleName();
     }
+
+    @Override
+    public HostTagResponse newHostTagResponse(HostTagVO tag) {
+        HostTagResponse tagResponse = new HostTagResponse();
+
+        tagResponse.setName(tag.getTag());
+        tagResponse.setHostId(tag.getHostId());
+        tagResponse.setImplicit(tag.getIsImplicit());
+
+        tagResponse.setObjectName("hosttag");
+
+        return tagResponse;
+    }
+
+    @Override
+    public List<HostTagVO> searchByIds(Long... tagIds) {
+        String batchCfg = _configDao.getValue("detail.batch.query.size");
+
+        final int detailsBatchSize = batchCfg != null ? Integer.parseInt(batchCfg) : 2000;
+
+        // query details by batches
+        List<HostTagVO> tagList = new ArrayList<>();
+        int curr_index = 0;
+
+        if (tagIds.length > detailsBatchSize) {
+            while ((curr_index + detailsBatchSize) <= tagIds.length) {
+                Long[] ids = new Long[detailsBatchSize];
+
+                for (int k = 0, j = curr_index; j < curr_index + detailsBatchSize; j++, k++) {
+                    ids[k] = tagIds[j];
+                }
+
+                SearchCriteria<HostTagVO> sc = stSearch.create();
+
+                sc.setParameters("idIN", (Object[])ids);
+
+                List<HostTagVO> vms = searchIncludingRemoved(sc, null, null, false);
+
+                if (vms != null) {
+                    tagList.addAll(vms);
+                }
+
+                curr_index += detailsBatchSize;
+            }
+        }
+
+        if (curr_index < tagIds.length) {
+            int batch_size = (tagIds.length - curr_index);
+            // set the ids value
+            Long[] ids = new Long[batch_size];
+
+            for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) {
+                ids[k] = tagIds[j];
+            }
+
+            SearchCriteria<HostTagVO> sc = stSearch.create();
+
+            sc.setParameters("idIN", (Object[])ids);
+
+            List<HostTagVO> tags = searchIncludingRemoved(sc, null, null, false);
+
+            if (tags != null) {
+                tagList.addAll(tags);
+            }
+        }
+
+        return tagList;
+    }
 }