Skip to content

Commit 119908c

Browse files
erikbocksdhslove
erikbocks
authored and
dhslove
committed
Usage parsers refactoring (apache#11097)
1 parent 4fd782c commit 119908c

19 files changed

+254
-616
lines changed

‎usage/src/main/java/com/cloud/usage/UsageManagerImpl.java‎

Lines changed: 8 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,9 @@
3434

3535
import com.cloud.network.Network;
3636
import com.cloud.usage.dao.UsageNetworksDao;
37-
import com.cloud.usage.parser.NetworksUsageParser;
37+
import com.cloud.usage.parser.UsageParser;
3838
import com.cloud.network.vpc.Vpc;
3939
import com.cloud.usage.dao.UsageVpcDao;
40-
import com.cloud.usage.parser.VpcUsageParser;
4140
import javax.inject.Inject;
4241
import javax.naming.ConfigurationException;
4342
import javax.persistence.EntityExistsException;
@@ -75,21 +74,6 @@
7574
import com.cloud.usage.dao.UsageVPNUserDao;
7675
import com.cloud.usage.dao.UsageVmDiskDao;
7776
import com.cloud.usage.dao.UsageVolumeDao;
78-
import com.cloud.usage.parser.BackupUsageParser;
79-
import com.cloud.usage.parser.BucketUsageParser;
80-
import com.cloud.usage.parser.IPAddressUsageParser;
81-
import com.cloud.usage.parser.LoadBalancerUsageParser;
82-
import com.cloud.usage.parser.NetworkOfferingUsageParser;
83-
import com.cloud.usage.parser.NetworkUsageParser;
84-
import com.cloud.usage.parser.PortForwardingUsageParser;
85-
import com.cloud.usage.parser.SecurityGroupUsageParser;
86-
import com.cloud.usage.parser.StorageUsageParser;
87-
import com.cloud.usage.parser.VMInstanceUsageParser;
88-
import com.cloud.usage.parser.VMSnapshotOnPrimaryParser;
89-
import com.cloud.usage.parser.VMSnapshotUsageParser;
90-
import com.cloud.usage.parser.VPNUserUsageParser;
91-
import com.cloud.usage.parser.VmDiskUsageParser;
92-
import com.cloud.usage.parser.VolumeUsageParser;
9377
import com.cloud.user.Account;
9478
import com.cloud.user.AccountVO;
9579
import com.cloud.user.UserStatisticsVO;
@@ -180,6 +164,9 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
180164
@Inject
181165
private UsageVpcDao usageVpcDao;
182166

167+
@Inject
168+
private List<UsageParser> usageParsers;
169+
183170
private String _version = null;
184171
private final Calendar _jobExecTime = Calendar.getInstance();
185172
private int _aggregationDuration = 0;
@@ -198,6 +185,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
198185
private Future _heartbeat = null;
199186
private Future _sanity = null;
200187
private boolean usageSnapshotSelection = false;
188+
201189
private static TimeZone usageAggregationTimeZone = TimeZone.getTimeZone("GMT");
202190

203191
public UsageManagerImpl() {
@@ -956,114 +944,12 @@ public void parse(UsageJobVO job, long startDateMillis, long endDateMillis) {
956944
private boolean parseHelperTables(AccountVO account, Date currentStartDate, Date currentEndDate) {
957945
boolean parsed = false;
958946

959-
parsed = VMInstanceUsageParser.parse(account, currentStartDate, currentEndDate);
960-
if (logger.isDebugEnabled()) {
961-
if (!parsed) {
962-
logger.debug("vm usage instances successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
963-
}
964-
}
947+
for (UsageParser parser : usageParsers) {
948+
parsed = parser.doParsing(account, currentStartDate, currentEndDate);
965949

966-
parsed = NetworkUsageParser.parse(account, currentStartDate, currentEndDate);
967-
if (logger.isDebugEnabled()) {
968-
if (!parsed) {
969-
logger.debug("network usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
970-
}
950+
logger.debug("{} usage was {} parsed for [{}].", parser.getParserName(), parsed ? "successfully" : "not successfully", account);
971951
}
972952

973-
parsed = VmDiskUsageParser.parse(account, currentStartDate, currentEndDate);
974-
if (logger.isDebugEnabled()) {
975-
if (!parsed) {
976-
logger.debug("vm disk usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
977-
}
978-
}
979-
980-
parsed = VolumeUsageParser.parse(account, currentStartDate, currentEndDate);
981-
if (logger.isDebugEnabled()) {
982-
if (!parsed) {
983-
logger.debug("volume usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
984-
}
985-
}
986-
987-
parsed = StorageUsageParser.parse(account, currentStartDate, currentEndDate);
988-
if (logger.isDebugEnabled()) {
989-
if (!parsed) {
990-
logger.debug("storage usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
991-
}
992-
}
993-
994-
parsed = SecurityGroupUsageParser.parse(account, currentStartDate, currentEndDate);
995-
if (logger.isDebugEnabled()) {
996-
if (!parsed) {
997-
logger.debug("Security Group usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
998-
}
999-
}
1000-
1001-
parsed = LoadBalancerUsageParser.parse(account, currentStartDate, currentEndDate);
1002-
if (logger.isDebugEnabled()) {
1003-
if (!parsed) {
1004-
logger.debug("load balancer usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1005-
}
1006-
}
1007-
1008-
parsed = PortForwardingUsageParser.parse(account, currentStartDate, currentEndDate);
1009-
if (logger.isDebugEnabled()) {
1010-
if (!parsed) {
1011-
logger.debug("port forwarding usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1012-
}
1013-
}
1014-
1015-
parsed = NetworkOfferingUsageParser.parse(account, currentStartDate, currentEndDate);
1016-
if (logger.isDebugEnabled()) {
1017-
if (!parsed) {
1018-
logger.debug("network offering usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1019-
}
1020-
}
1021-
1022-
parsed = IPAddressUsageParser.parse(account, currentStartDate, currentEndDate);
1023-
if (logger.isDebugEnabled()) {
1024-
if (!parsed) {
1025-
logger.debug("IPAddress usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1026-
}
1027-
}
1028-
parsed = VPNUserUsageParser.parse(account, currentStartDate, currentEndDate);
1029-
if (logger.isDebugEnabled()) {
1030-
if (!parsed) {
1031-
logger.debug("VPN user usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1032-
}
1033-
}
1034-
parsed = VMSnapshotUsageParser.parse(account, currentStartDate, currentEndDate);
1035-
if (logger.isDebugEnabled()) {
1036-
if (!parsed) {
1037-
logger.debug("VM Snapshot usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1038-
}
1039-
}
1040-
parsed = VMSnapshotOnPrimaryParser.parse(account, currentStartDate, currentEndDate);
1041-
if (logger.isDebugEnabled()) {
1042-
if (!parsed) {
1043-
logger.debug("VM Snapshot on primary usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1044-
}
1045-
}
1046-
parsed = BackupUsageParser.parse(account, currentStartDate, currentEndDate);
1047-
if (logger.isDebugEnabled()) {
1048-
if (!parsed) {
1049-
logger.debug("VM Backup usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1050-
}
1051-
}
1052-
parsed = BucketUsageParser.parse(account, currentStartDate, currentEndDate);
1053-
if (logger.isDebugEnabled()) {
1054-
if (!parsed) {
1055-
logger.debug("Bucket usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")");
1056-
}
1057-
}
1058-
parsed = NetworksUsageParser.parse(account, currentStartDate, currentEndDate);
1059-
if (!parsed) {
1060-
logger.debug("Networks usage not parsed for account [{}}].", account);
1061-
}
1062-
1063-
parsed = VpcUsageParser.parse(account, currentStartDate, currentEndDate);
1064-
if (!parsed) {
1065-
logger.debug(String.format("VPC usage failed to parse for account [%s].", account));
1066-
}
1067953
return parsed;
1068954
}
1069955

‎usage/src/main/java/com/cloud/usage/parser/BackupUsageParser.java‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,49 +21,35 @@
2121
import java.util.Date;
2222
import java.util.List;
2323

24-
import javax.annotation.PostConstruct;
2524
import javax.inject.Inject;
2625

2726
import org.apache.cloudstack.usage.UsageTypes;
28-
import org.apache.logging.log4j.Logger;
29-
import org.apache.logging.log4j.LogManager;
3027
import org.springframework.stereotype.Component;
3128

3229
import com.cloud.usage.UsageBackupVO;
3330
import com.cloud.usage.UsageVO;
34-
import com.cloud.usage.dao.UsageDao;
3531
import com.cloud.usage.dao.UsageBackupDao;
3632
import com.cloud.user.AccountVO;
3733

3834
@Component
39-
public class BackupUsageParser {
40-
protected static Logger LOGGER = LogManager.getLogger(BackupUsageParser.class);
41-
42-
private static UsageDao s_usageDao;
43-
private static UsageBackupDao s_usageBackupDao;
44-
45-
@Inject
46-
private UsageDao usageDao;
35+
public class BackupUsageParser extends UsageParser {
4736
@Inject
4837
private UsageBackupDao usageBackupDao;
4938

50-
@PostConstruct
51-
void init() {
52-
s_usageDao = usageDao;
53-
s_usageBackupDao = usageBackupDao;
39+
@Override
40+
public String getParserName() {
41+
return "VM Backup";
5442
}
5543

56-
public static boolean parse(AccountVO account, Date startDate, Date endDate) {
57-
if (LOGGER.isDebugEnabled()) {
58-
LOGGER.debug("Parsing all VM Backup usage events for account: " + account.getId());
59-
}
44+
@Override
45+
protected boolean parse(AccountVO account, Date startDate, Date endDate) {
6046
if ((endDate == null) || endDate.after(new Date())) {
6147
endDate = new Date();
6248
}
6349

64-
final List<UsageBackupVO> usageBackups = s_usageBackupDao.getUsageRecords(account.getId(), startDate, endDate);
50+
final List<UsageBackupVO> usageBackups = usageBackupDao.getUsageRecords(account.getId(), startDate, endDate);
6551
if (usageBackups == null || usageBackups.isEmpty()) {
66-
LOGGER.debug("No VM Backup usage for this period");
52+
logger.debug("No VM Backup usage for this period");
6753
return true;
6854
}
6955

@@ -91,7 +77,7 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) {
9177
new UsageVO(zoneId, account.getAccountId(), account.getDomainId(), description, usageDisplay + " Hrs",
9278
UsageTypes.BACKUP, new Double(usage), vmId, null, offeringId, null, vmId,
9379
usageBackup.getSize(), usageBackup.getProtectedSize(), startDate, endDate);
94-
s_usageDao.persist(usageRecord);
80+
usageDao.persist(usageRecord);
9581
}
9682

9783
return true;

‎usage/src/main/java/com/cloud/usage/parser/BucketUsageParser.java‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,47 +19,32 @@
1919
import com.cloud.usage.BucketStatisticsVO;
2020
import com.cloud.usage.UsageVO;
2121
import com.cloud.usage.dao.BucketStatisticsDao;
22-
import com.cloud.usage.dao.UsageDao;
2322
import com.cloud.user.AccountVO;
2423
import org.apache.cloudstack.usage.UsageTypes;
25-
import org.apache.logging.log4j.LogManager;
26-
import org.apache.logging.log4j.Logger;
2724
import org.springframework.stereotype.Component;
2825

29-
import javax.annotation.PostConstruct;
3026
import javax.inject.Inject;
3127
import java.util.ArrayList;
3228
import java.util.Date;
3329
import java.util.List;
3430

3531
@Component
36-
public class BucketUsageParser {
37-
public static final Logger LOGGER = LogManager.getLogger(BucketUsageParser.class);
38-
39-
private static UsageDao s_usageDao;
40-
private static BucketStatisticsDao s_bucketStatisticsDao;
41-
42-
@Inject
43-
private UsageDao _usageDao;
32+
public class BucketUsageParser extends UsageParser {
4433
@Inject
45-
private BucketStatisticsDao _bucketStatisticsDao;
34+
private BucketStatisticsDao bucketStatisticsDao;
4635

47-
@PostConstruct
48-
void init() {
49-
s_usageDao = _usageDao;
50-
s_bucketStatisticsDao = _bucketStatisticsDao;
36+
@Override
37+
public String getParserName() {
38+
return "Bucket";
5139
}
5240

53-
public static boolean parse(AccountVO account, Date startDate, Date endDate) {
54-
if (LOGGER.isDebugEnabled()) {
55-
LOGGER.debug("Parsing all Bucket usage events for account {}", account);
56-
}
57-
41+
@Override
42+
protected boolean parse(AccountVO account, Date startDate, Date endDate) {
5843
if ((endDate == null) || endDate.after(new Date())) {
5944
endDate = new Date();
6045
}
6146

62-
List<BucketStatisticsVO> BucketStatisticsVOs = s_bucketStatisticsDao.listBy(account.getId());
47+
List<BucketStatisticsVO> BucketStatisticsVOs = bucketStatisticsDao.listBy(account.getId());
6348

6449
List<UsageVO> usageRecords = new ArrayList<>();
6550
for (BucketStatisticsVO bucketStatistics : BucketStatisticsVOs) {
@@ -72,7 +57,7 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) {
7257
}
7358
}
7459

75-
s_usageDao.saveUsageRecords(usageRecords);
60+
usageDao.saveUsageRecords(usageRecords);
7661

7762
return true;
7863
}

0 commit comments

Comments
 (0)