Merge branch '4.20'

Author: harikrishna-patnala

agent/src/main/java/com/cloud/agent/Agent.java

@@ -613,7 +613,7 @@ protected void setupStartupCommand(final StartupCommand startup) {
     }
 
     protected String getAgentArch() {
-        String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 1000);
+        String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 2000);
         logger.debug("Arch for agent: {} found: {}", _name, arch);
         return arch;
     }

api/src/main/java/com/cloud/server/ManagementService.java

@@ -20,7 +20,6 @@
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.user.UserData;
 import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd;
 import org.apache.cloudstack.api.command.admin.config.ListCfgGroupsByCmd;
 import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd;
@@ -72,6 +71,7 @@
 import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd;
 import org.apache.cloudstack.config.Configuration;
 import org.apache.cloudstack.config.ConfigurationGroup;
+import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.alert.Alert;
 import com.cloud.capacity.Capacity;
@@ -91,6 +91,7 @@
 import com.cloud.storage.GuestOsCategory;
 import com.cloud.storage.StoragePool;
 import com.cloud.user.SSHKeyPair;
+import com.cloud.user.UserData;
 import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
 import com.cloud.vm.InstanceGroup;
@@ -104,6 +105,14 @@
 public interface ManagementService {
     static final String Name = "management-server";
 
+    ConfigKey<Boolean> JsInterpretationEnabled = new ConfigKey<>("Hidden"
+            , Boolean.class
+            , "js.interpretation.enabled"
+            , "false"
+            , "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules."
+            , false
+            , ConfigKey.Scope.Global);
+
     /**
      * returns the a map of the names/values in the configuration table
      *
@@ -509,4 +518,6 @@ VirtualMachine upgradeSystemVM(ScaleSystemVMCmd cmd) throws ResourceUnavailableE
 
     boolean removeManagementServer(RemoveManagementServerCmd cmd);
 
+    void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue);
+
 }

api/src/main/java/com/cloud/server/ResourceManagerUtil.java

@@ -18,6 +18,7 @@
 
 public interface ResourceManagerUtil {
     long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType);
+    long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType, boolean checkAccess);
     String getUuid(String resourceId, ResourceTag.ResourceObjectType resourceType);
     ResourceTag.ResourceObjectType getResourceType(String resourceTypeStr);
     void checkResourceAccessible(Long accountId, Long domainId, String exceptionMessage);

engine/components-api/src/main/java/com/cloud/template/TemplateManager.java

@@ -57,6 +57,13 @@ public interface TemplateManager {
             + "will validate if the provided URL is resolvable during the register of templates/ISOs before persisting them in the database.",
             true);
 
+    ConfigKey<Boolean> TemplateDeleteFromPrimaryStorage = new ConfigKey<Boolean>("Advanced",
+            Boolean.class,
+            "template.delete.from.primary.storage", "true",
+            "Template when deleted will be instantly deleted from the Primary Storage",
+            true,
+            ConfigKey.Scope.Global);
+
     static final String VMWARE_TOOLS_ISO = "vmware-tools.iso";
     static final String XS_TOOLS_ISO = "xs-tools.iso";
 
@@ -104,6 +111,8 @@ public interface TemplateManager {
      */
     List<VMTemplateStoragePoolVO> getUnusedTemplatesInPool(StoragePoolVO pool);
 
+    void evictTemplateFromStoragePoolsForZones(Long templateId, List<Long> zoneId);
+
     /**
      * Deletes a template in the specified storage pool.
      *

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -4799,6 +4799,18 @@ public NicVO doInTransaction(TransactionStatus status) {
             }
         });
 
+        if (selectedIp != null && GuestType.Shared.equals(network.getGuestType())) {
+            IPAddressVO ipAddressVO = _ipAddressDao.findByIpAndSourceNetworkId(network.getId(), selectedIp);
+            if (ipAddressVO != null && IpAddress.State.Free.equals(ipAddressVO.getState())) {
+                ipAddressVO.setState(IPAddressVO.State.Allocated);
+                ipAddressVO.setAllocatedTime(new Date());
+                Account account = _accountDao.findById(vm.getAccountId());
+                ipAddressVO.setAllocatedInDomainId(account.getDomainId());
+                ipAddressVO.setAllocatedToAccountId(account.getId());
+                _ipAddressDao.update(ipAddressVO.getId(), ipAddressVO);
+            }
+        }
+
         final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId());
         final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),
                 _networkModel.getNetworkTag(vm.getHypervisorType(), network));
@@ -4810,15 +4822,15 @@ protected String getSelectedIpForNicImport(Network network, DataCenter dataCente
         if (network.getGuestType() == GuestType.L2) {
             return null;
         }
-        return dataCenter.getNetworkType() == NetworkType.Basic ?
-                getSelectedIpForNicImportOnBasicZone(ipAddresses.getIp4Address(), network, dataCenter):
+        return GuestType.Shared.equals(network.getGuestType()) ?
+                getSelectedIpForNicImportOnSharedNetwork(ipAddresses.getIp4Address(), network, dataCenter):
                 _ipAddrMgr.acquireGuestIpAddress(network, ipAddresses.getIp4Address());
     }
 
-    protected String getSelectedIpForNicImportOnBasicZone(String requestedIp, Network network, DataCenter dataCenter) {
+    protected String getSelectedIpForNicImportOnSharedNetwork(String requestedIp, Network network, DataCenter dataCenter) {
         IPAddressVO ipAddressVO = StringUtils.isBlank(requestedIp) ?
                 _ipAddressDao.findBySourceNetworkIdAndDatacenterIdAndState(network.getId(), dataCenter.getId(), IpAddress.State.Free):
-                _ipAddressDao.findByIp(requestedIp);
+                _ipAddressDao.findByIpAndSourceNetworkId(network.getId(), requestedIp);
         if (ipAddressVO == null || ipAddressVO.getState() != IpAddress.State.Free) {
             String msg = String.format("Cannot find a free IP to assign to VM NIC on network %s", network.getName());
             logger.error(msg);

engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java

@@ -822,7 +822,7 @@ public void testGetGuestIpForNicImportBasicZoneManualIP() {
         Mockito.when(network.getId()).thenReturn(networkId);
         Mockito.when(dataCenter.getId()).thenReturn(dataCenterId);
         Mockito.when(ipAddresses.getIp4Address()).thenReturn(requestedIp);
-        Mockito.when(testOrchestrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO);
+        Mockito.when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(networkId, requestedIp)).thenReturn(ipAddressVO);
         String ipAddress = testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses);
         Assert.assertEquals(requestedIp, ipAddress);
     }

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java

@@ -35,6 +35,8 @@ public interface VMTemplatePoolDao extends GenericDao<VMTemplateStoragePoolVO, L
 
     List<VMTemplateStoragePoolVO> listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state);
 
+    List<VMTemplateStoragePoolVO> listByPoolIdsAndTemplate(List<Long> poolIds, Long templateId);
+
     List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState);
 
     List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId);

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java

@@ -150,6 +150,16 @@ public VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId,
         return findOneIncludingRemovedBy(sc);
     }
 
+    @Override
+    public List<VMTemplateStoragePoolVO> listByPoolIdsAndTemplate(List<Long> poolIds, Long templateId) {
+        SearchCriteria<VMTemplateStoragePoolVO> sc = PoolTemplateSearch.create();
+        if (CollectionUtils.isNotEmpty(poolIds)) {
+            sc.setParameters("pool_id", poolIds.toArray());
+        }
+        sc.setParameters("template_id", templateId);
+        return listBy(sc);
+    }
+
     @Override
     public List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState) {
         SearchCriteria<VMTemplateStoragePoolVO> sc = TemplateStatusSearch.create();

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -171,4 +171,5 @@ Pair<List<Long>, Integer> searchForIdsAndCount(Long storagePoolId, String storag
 
     List<StoragePoolVO> findPoolsByStorageTypeAndZone(Storage.StoragePoolType storageType, Long zoneId);
 
+    List<StoragePoolVO> listByDataCenterIds(List<Long> dataCenterIds);
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -65,6 +65,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
     private final GenericSearchBuilder<StoragePoolVO, Long> StatusCountSearch;
     private final SearchBuilder<StoragePoolVO> ClustersSearch;
     private final SearchBuilder<StoragePoolVO> IdsSearch;
+    private final SearchBuilder<StoragePoolVO> DcsSearch;
 
     @Inject
     private StoragePoolDetailsDao _detailsDao;
@@ -167,6 +168,9 @@ public PrimaryDataStoreDaoImpl() {
         IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
         IdsSearch.done();
 
+        DcsSearch = createSearchBuilder();
+        DcsSearch.and("dataCenterId", DcsSearch.entity().getDataCenterId(), SearchCriteria.Op.IN);
+        DcsSearch.done();
     }
 
     @Override
@@ -927,6 +931,16 @@ public List<StoragePoolVO> findPoolsByStorageTypeAndZone(Storage.StoragePoolType
         return listBy(sc);
     }
 
+    @Override
+    public List<StoragePoolVO> listByDataCenterIds(List<Long> dataCenterIds) {
+        if (CollectionUtils.isEmpty(dataCenterIds)) {
+            return Collections.emptyList();
+        }
+        SearchCriteria<StoragePoolVO> sc = DcsSearch.create();
+        sc.setParameters("dataCenterId", dataCenterIds.toArray());
+        return listBy(sc);
+    }
+
     private SearchCriteria<StoragePoolVO> createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName,
                                                                           Long zoneId, String path, Long podId, Long clusterId, Long hostId, String address, ScopeType scopeType,
                                                                           StoragePoolStatus status, String keyword, String storageAccessGroup) {