Backport #9888 to 4.19: Fix Usage inconsistencies

engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDao.java

@@ -28,4 +28,6 @@ public interface UsageNetworksDao extends GenericDao<UsageNetworksVO, Long> {
     void remove(long networkId, Date removed);
 
     List<UsageNetworksVO> getUsageRecords(Long accountId, Date startDate, Date endDate);
+
+    List<UsageNetworksVO> listAll(long networkId);
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java

@@ -16,15 +16,16 @@
 // under the License.
 package com.cloud.usage.dao;
 
-import com.cloud.network.Network;
 import com.cloud.usage.UsageNetworksVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.ArrayList;
@@ -39,6 +40,14 @@
             " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " +
             " OR ((created <= ?) AND (removed >= ?)))";
 
+    private SearchBuilder<UsageNetworksVO> usageNetworksSearch;
+
+    @PostConstruct
+    public void init() {
+        usageNetworksSearch = createSearchBuilder();
+        usageNetworksSearch.and("networkId", usageNetworksSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
+        usageNetworksSearch.done();
+    }
 
     @Override
     public void update(long networkId, long newNetworkOffering, String state) {
@@ -68,11 +77,10 @@
             SearchCriteria<UsageNetworksVO> sc = this.createSearchCriteria();
             sc.addAnd("networkId", SearchCriteria.Op.EQ, networkId);
             sc.addAnd("removed", SearchCriteria.Op.NULL);
-            UsageNetworksVO vo = findOneBy(sc);
-            if (vo != null) {
-                vo.setRemoved(removed);
-                vo.setState(Network.State.Destroy.name());
-                update(vo.getId(), vo);
+            List<UsageNetworksVO> usageNetworksVOs = listBy(sc);
+            for (UsageNetworksVO entry : usageNetworksVOs) {
+                entry.setRemoved(removed);
+                update(entry.getId(), entry);
             }
         } catch (final Exception e) {
             txn.rollback();
@@ -131,4 +139,11 @@
 
         return usageRecords;
     }
+
+    @Override
+    public List<UsageNetworksVO> listAll(long networkId) {
+        SearchCriteria<UsageNetworksVO> sc = usageNetworksSearch.create();
+        sc.setParameters("networkId", networkId);
+        return listBy(sc);
+    }
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDao.java

@@ -24,6 +24,10 @@
 
 public interface UsageVpcDao extends GenericDao<UsageVpcVO, Long> {
     void update(UsageVpcVO usage);
+
     void remove(long vpcId, Date removed);
+
     List<UsageVpcVO> getUsageRecords(Long accountId, Date startDate, Date endDate);
+
+    List<UsageVpcVO> listAll(long vpcId);
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java

@@ -16,15 +16,16 @@
 // under the License.
 package com.cloud.usage.dao;
 
-import com.cloud.network.vpc.Vpc;
 import com.cloud.usage.UsageVpcVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.ArrayList;
@@ -39,6 +40,15 @@
             " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " +
             " OR ((created <= ?) AND (removed >= ?)))";
 
+    private SearchBuilder<UsageVpcVO> usageVpcSearch;
+
+    @PostConstruct
+    public void init() {
+        usageVpcSearch = createSearchBuilder();
+        usageVpcSearch.and("vpcId", usageVpcSearch.entity().getVpcId(), SearchCriteria.Op.EQ);
+        usageVpcSearch.done();
+    }
+
     @Override
     public void update(UsageVpcVO usage) {
         TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
@@ -66,11 +76,10 @@
             SearchCriteria<UsageVpcVO> sc = this.createSearchCriteria();
             sc.addAnd("vpcId", SearchCriteria.Op.EQ, vpcId);
             sc.addAnd("removed", SearchCriteria.Op.NULL);
-            UsageVpcVO vo = findOneBy(sc);
-            if (vo != null) {
-                vo.setRemoved(removed);
-                vo.setState(Vpc.State.Inactive.name());
-                update(vo.getId(), vo);
+            List<UsageVpcVO> usageVpcVOs = listBy(sc);
+            for (UsageVpcVO entry : usageVpcVOs) {
+                entry.setRemoved(removed);
+                update(entry.getId(), entry);
             }
         } catch (final Exception e) {
             txn.rollback();
@@ -128,4 +137,11 @@
 
         return usageRecords;
     }
+
+    @Override
+    public List<UsageVpcVO> listAll(long vpcId) {
+        SearchCriteria<UsageVpcVO> sc = usageVpcSearch.create();
+        sc.setParameters("vpcId", vpcId);
+        return listBy(sc);
+    }
 }

usage/src/main/java/com/cloud/usage/UsageManagerImpl.java

@@ -1540,7 +1540,7 @@
             //For volumes which are 'attached' successfully, set the 'deleted' column in the usage_storage table,
             //so that the secondary storage should stop accounting and only primary will be accounted.
             SearchCriteria<UsageStorageVO> sc = _usageStorageDao.createSearchCriteria();
-            sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+            sc.addAnd("entityId", SearchCriteria.Op.EQ, volId);
             sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME);
             List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc, null);
             if (volumesVOs != null) {
@@ -1595,7 +1595,8 @@
             //For Upload event add an entry to the usage_storage table.
             SearchCriteria<UsageStorageVO> sc = _usageStorageDao.createSearchCriteria();
             sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
-            sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+            sc.addAnd("entityId", SearchCriteria.Op.EQ, volId);
+            sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME);
             sc.addAnd("deleted", SearchCriteria.Op.NULL);
             List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc, null);
 
@@ -1772,7 +1773,7 @@
         } else if (EventTypes.EVENT_LOAD_BALANCER_DELETE.equals(event.getType())) {
             SearchCriteria<UsageLoadBalancerPolicyVO> sc = _usageLoadBalancerPolicyDao.createSearchCriteria();
             sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
-            sc.addAnd("id", SearchCriteria.Op.EQ, id);
+            sc.addAnd("lbId", SearchCriteria.Op.EQ, id);
             sc.addAnd("deleted", SearchCriteria.Op.NULL);
             List<UsageLoadBalancerPolicyVO> lbVOs = _usageLoadBalancerPolicyDao.search(sc, null);
             if (lbVOs.size() > 1) {
@@ -1806,7 +1807,7 @@
         } else if (EventTypes.EVENT_NET_RULE_DELETE.equals(event.getType())) {
             SearchCriteria<UsagePortForwardingRuleVO> sc = _usagePortForwardingRuleDao.createSearchCriteria();
             sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
-            sc.addAnd("id", SearchCriteria.Op.EQ, id);
+            sc.addAnd("pfId", SearchCriteria.Op.EQ, id);
             sc.addAnd("deleted", SearchCriteria.Op.NULL);
             List<UsagePortForwardingRuleVO> pfVOs = _usagePortForwardingRuleDao.search(sc, null);
             if (pfVOs.size() > 1) {
@@ -2104,7 +2105,7 @@
         } else if (EventTypes.EVENT_VM_SNAPSHOT_OFF_PRIMARY.equals(event.getType())) {
             QueryBuilder<UsageSnapshotOnPrimaryVO> sc = QueryBuilder.create(UsageSnapshotOnPrimaryVO.class);
             sc.and(sc.entity().getAccountId(), SearchCriteria.Op.EQ, event.getAccountId());
-            sc.and(sc.entity().getId(), SearchCriteria.Op.EQ, vmId);
+            sc.and(sc.entity().getVmId(), SearchCriteria.Op.EQ, vmId);
             sc.and(sc.entity().getName(), SearchCriteria.Op.EQ, name);
             sc.and(sc.entity().getDeleted(), SearchCriteria.Op.NULL);
             List<UsageSnapshotOnPrimaryVO> vmsnaps = sc.list();
@@ -2142,15 +2143,29 @@
     }
 
     private void handleNetworkEvent(UsageEventVO event) {
+        long networkId = event.getResourceId();
         Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId());
         long domainId = account.getDomainId();
         if (EventTypes.EVENT_NETWORK_DELETE.equals(event.getType())) {
-            usageNetworksDao.remove(event.getResourceId(), event.getCreateDate());
+            usageNetworksDao.remove(networkId, event.getCreateDate());
         } else if (EventTypes.EVENT_NETWORK_CREATE.equals(event.getType())) {
-            UsageNetworksVO usageNetworksVO = new UsageNetworksVO(event.getResourceId(), event.getOfferingId(), event.getZoneId(), event.getAccountId(), domainId, Network.State.Allocated.name(), event.getCreateDate(), null);
+            List<UsageNetworksVO> entries = usageNetworksDao.listAll(networkId);
+            if (!entries.isEmpty()) {
+                s_logger.warn(String.format("Received a NETWORK.CREATE event for a network [%s] that already has helper entries; " +
+                                "therefore, we will not create a new one.", networkId));
+                return;
+            }
+            s_logger.debug(String.format("Creating a helper entry for network [%s].", networkId));
+            UsageNetworksVO usageNetworksVO = new UsageNetworksVO(networkId, event.getOfferingId(), event.getZoneId(),
+                    event.getAccountId(), domainId, Network.State.Allocated.name(), event.getCreateDate(), null);
             usageNetworksDao.persist(usageNetworksVO);
         } else if (EventTypes.EVENT_NETWORK_UPDATE.equals(event.getType())) {
-            usageNetworksDao.update(event.getResourceId(), event.getOfferingId(), event.getResourceType());
+            s_logger.debug(String.format("Marking previous helper entries of network [%s] as removed.", networkId));
+            usageNetworksDao.remove(networkId, event.getCreateDate());
+            s_logger.debug(String.format("Creating an updated helper entry for network [%s].", networkId));
+            UsageNetworksVO usageNetworksVO = new UsageNetworksVO(networkId, event.getOfferingId(), event.getZoneId(),
+                    event.getAccountId(), domainId, event.getResourceType(), event.getCreateDate(), null);
+            usageNetworksDao.persist(usageNetworksVO);
         } else {
             s_logger.error(String.format("Unknown event type [%s] in Networks event parser. Skipping it.", event.getType()));
         }
@@ -2162,6 +2177,13 @@
         if (EventTypes.EVENT_VPC_DELETE.equals(event.getType())) {
             usageVpcDao.remove(event.getResourceId(), event.getCreateDate());
         } else if (EventTypes.EVENT_VPC_CREATE.equals(event.getType())) {
+            List<UsageVpcVO> entries = usageVpcDao.listAll(event.getResourceId());
+            if (!entries.isEmpty()) {
+                s_logger.warn(String.format("Active helper entries already exist for VPC [%s]; therefore, we will not create a new one.",
+                        event.getResourceId()));
+                return;
+            }
+            s_logger.debug(String.format("Creating a helper entry for VPC [%s].", event.getResourceId()));
             UsageVpcVO usageVPCVO = new UsageVpcVO(event.getResourceId(), event.getZoneId(), event.getAccountId(), domainId, Vpc.State.Enabled.name(), event.getCreateDate(), null);
             usageVpcDao.persist(usageVPCVO);
         } else {