chore: add csv data plotter

fabianbormann · fabianbormann · commit 9f1554c2e9c5 · 2024-05-03T21:32:43.000+02:00
diff --git a/calculation/src/main/java/org/cardanofoundation/rewards/calculation/EpochCalculation.java b/calculation/src/main/java/org/cardanofoundation/rewards/calculation/EpochCalculation.java
@@ -80,6 +80,7 @@ public static EpochCalculationResult calculateEpochRewardPots(final int epoch,
 
         // The sum of all the refunds attached to unregistered reward accounts are added to the
         // treasury (see: Pool Reap Transition, p.53, figure 40, shelley-ledger.pdf)
+        BigInteger unclaimedRefunds = BigInteger.ZERO;
         if (rewardAddressesOfRetiredPools.size() > 0) {
             List<String> deregisteredOwnerAccounts = deregisteredAccountsOnEpochBoundary.stream()
                     .filter(rewardAddressesOfRetiredPools::contains).toList();
@@ -95,6 +96,7 @@ public static EpochCalculationResult calculateEpochRewardPots(final int epoch,
                     // If the reward address has been unregistered, the deposit can not be returned
                     // and will be added to the treasury instead (Pool Reap see: shelley-ledger.pdf p.53)
                     treasuryForCurrentEpoch = treasuryForCurrentEpoch.add(POOL_DEPOSIT_IN_LOVELACE);
+                    unclaimedRefunds = unclaimedRefunds.add(POOL_DEPOSIT_IN_LOVELACE);
                 }
             }
         }
@@ -185,6 +187,7 @@ public static EpochCalculationResult calculateEpochRewardPots(final int epoch,
                 .totalRewardPot(rewardPot)
                 .treasuryWithdrawals(treasuryWithdrawals)
                 .unspendableEarnedRewards(unspendableEarnedRewards)
+                .unclaimedRefunds(unclaimedRefunds)
                 .build();
 
         epochCalculationResult.setTotalDistributedRewards(totalDistributedRewards);
diff --git a/calculation/src/main/java/org/cardanofoundation/rewards/calculation/TreasuryCalculation.java b/calculation/src/main/java/org/cardanofoundation/rewards/calculation/TreasuryCalculation.java
@@ -31,6 +31,7 @@ public static TreasuryCalculationResult calculateTreasuryInEpoch(int epoch, Prot
               .totalRewardPot(BigInteger.ZERO)
               .treasuryWithdrawals(BigInteger.ZERO)
               .unspendableEarnedRewards(BigInteger.ZERO)
+              .unclaimedRefunds(BigInteger.ZERO)
               .build();
     }
 
@@ -58,13 +59,14 @@ public static TreasuryCalculationResult calculateTreasuryInEpoch(int epoch, Prot
     final BigInteger treasuryCut = multiplyAndFloor(totalRewardPot, treasuryGrowthRate);
     BigInteger treasuryForCurrentEpoch = treasuryInPreviousEpoch.add(treasuryCut);
 
+    BigInteger unclaimedRefunds = BigInteger.ZERO;
     if (rewardAddressesOfRetiredPools.size() > 0) {
       HashSet<String> deregisteredRewardAccounts = deregisteredAccounts.stream()
               .filter(rewardAddressesOfRetiredPools::contains).collect(Collectors.toCollection(HashSet::new));
       List<String> ownerAccountsRegisteredInThePast = registeredAccountsUntilNow.stream()
               .filter(rewardAddressesOfRetiredPools::contains).toList();
 
-      BigInteger unclaimedRefunds = calculateUnclaimedRefundsForRetiredPools(rewardAddressesOfRetiredPools, deregisteredRewardAccounts, ownerAccountsRegisteredInThePast);
+      unclaimedRefunds = calculateUnclaimedRefundsForRetiredPools(rewardAddressesOfRetiredPools, deregisteredRewardAccounts, ownerAccountsRegisteredInThePast);
       treasuryForCurrentEpoch = treasuryForCurrentEpoch.add(unclaimedRefunds);
     }
 
@@ -84,6 +86,7 @@ public static TreasuryCalculationResult calculateTreasuryInEpoch(int epoch, Prot
             .totalRewardPot(totalRewardPot)
             .treasuryWithdrawals(treasuryWithdrawals)
             .unspendableEarnedRewards(unspendableEarnedRewards)
+            .unclaimedRefunds(unclaimedRefunds)
             .build();
     }
 
diff --git a/calculation/src/main/java/org/cardanofoundation/rewards/calculation/domain/TreasuryCalculationResult.java b/calculation/src/main/java/org/cardanofoundation/rewards/calculation/domain/TreasuryCalculationResult.java
@@ -15,4 +15,5 @@ public class TreasuryCalculationResult {
     BigInteger totalRewardPot;
     BigInteger treasuryWithdrawals;
     BigInteger unspendableEarnedRewards;
+    BigInteger unclaimedRefunds;
 }
diff --git a/report/statistics.csv b/report/statistics.csv
diff --git a/validation/src/main/java/org/cardanofoundation/rewards/RewardsApplication.java b/validation/src/main/java/org/cardanofoundation/rewards/RewardsApplication.java
@@ -2,6 +2,7 @@
 
 import org.cardanofoundation.rewards.validation.data.fetcher.DbSyncDataFetcher;
 import org.cardanofoundation.rewards.validation.data.fetcher.KoiosDataFetcher;
+import org.cardanofoundation.rewards.validation.data.plotter.CsvDataPlotter;
 import org.cardanofoundation.rewards.validation.data.plotter.JsonDataPlotter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,6 +47,9 @@ public class RewardsApplication implements ApplicationRunner {
   @Autowired
   private JsonDataPlotter jsonDataPlotter;
 
+  @Autowired
+  private CsvDataPlotter csvDataPlotter;
+
   @Autowired
   private KoiosDataFetcher koiosDataFetcher;
 
@@ -89,7 +93,11 @@ public void run(ApplicationArguments args) throws Exception {
             }
           }
       } else if (runMode.equals("plot")) {
-        jsonDataPlotter.plot(startEpoch, endEpoch);
+          if (activeProfiles.contains("csv")) {
+              csvDataPlotter.plot(startEpoch, endEpoch);
+          } else {
+              jsonDataPlotter.plot(startEpoch, endEpoch);
+          }
       } else if (!runMode.equals("test")) {
           logger.warn("Unknown run mode: " + runMode);
       }
diff --git a/validation/src/main/java/org/cardanofoundation/rewards/validation/data/plotter/CsvDataPlotter.java b/validation/src/main/java/org/cardanofoundation/rewards/validation/data/plotter/CsvDataPlotter.java
@@ -0,0 +1,65 @@
+package org.cardanofoundation.rewards.validation.data.plotter;
+
+import org.cardanofoundation.rewards.calculation.domain.EpochCalculationResult;
+import org.cardanofoundation.rewards.calculation.domain.PoolState;
+import org.cardanofoundation.rewards.validation.EpochValidation;
+import org.cardanofoundation.rewards.validation.data.provider.JsonDataProvider;
+import org.cardanofoundation.rewards.validation.domain.EpochValidationInput;
+import org.cardanofoundation.rewards.validation.domain.TreasuryValidationResult;
+import org.cardanofoundation.rewards.validation.util.CsvConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Service
+public class CsvDataPlotter implements DataPlotter {
+    private static final Logger logger = LoggerFactory.getLogger(JsonDataPlotter.class);
+
+    @Autowired
+    private JsonDataProvider jsonDataProvider;
+
+    @Override
+    public void plot(int epochStart, int epochEnd) {
+        if (epochStart > epochEnd) {
+            throw new IllegalArgumentException("epochStart must be less than or equal to epochEnd");
+        }
+
+        Map<Integer, TreasuryValidationResult> epochTreasuryValidationResultMap = new LinkedHashMap<>();
+
+        List<HashMap<String, String>> data = new ArrayList<>();
+        for (int epoch = epochStart; epoch < epochEnd; epoch++) {
+            HashMap<String, String> row = new HashMap<>();
+            boolean detailedValidation = false;
+
+            EpochValidationInput epochValidationInput = jsonDataProvider.getEpochValidationInput(epoch);
+            HashSet<String> poolIds = epochValidationInput.getPoolStates().stream().map(PoolState::getPoolId).collect(Collectors.toCollection(HashSet::new));
+            EpochCalculationResult epochCalculationResult = EpochValidation.calculateEpochRewardPots(epoch,
+                    jsonDataProvider, detailedValidation);
+
+            row.put("epoch", String.valueOf(epoch));
+            row.put("reserves", String.valueOf(epochCalculationResult.getReserves()));
+            row.put("treasury", String.valueOf(epochCalculationResult.getTreasury()));
+            row.put("epoch_fees", String.valueOf(epochValidationInput.getFees()));
+            row.put("unspendable_earned_rewards", String.valueOf(epochCalculationResult.getTreasuryCalculationResult().getUnspendableEarnedRewards()));
+            row.put("undistributed_rewards", String.valueOf(epochCalculationResult.getTotalUndistributedRewards()));
+            row.put("unclaimed_refunds", String.valueOf(epochCalculationResult.getTreasuryCalculationResult().getUnclaimedRefunds()));
+            row.put("treasury_withdrawals", String.valueOf(epochCalculationResult.getTreasuryCalculationResult().getTreasuryWithdrawals()));
+            row.put("active_epoch_stake", String.valueOf(epochValidationInput.getActiveStake()));
+            row.put("block_count", String.valueOf(epochValidationInput.getBlockCount()));
+            row.put("unique_pools_with_blocks", String.valueOf(poolIds.size()));
+            data.add(row);
+        }
+
+        try {
+            CsvConverter.writeObjectToCsvFile(data,
+                    "./report/statistics.csv");
+        } catch (Exception e) {
+            logger.error(e.getMessage());
+            logger.warn("Failed to write statistics to csv file");
+        }
+    }
+}
diff --git a/validation/src/main/java/org/cardanofoundation/rewards/validation/util/CsvConverter.java b/validation/src/main/java/org/cardanofoundation/rewards/validation/util/CsvConverter.java
@@ -0,0 +1,39 @@
+package org.cardanofoundation.rewards.validation.util;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+
+public class CsvConverter {
+
+    public static void writeObjectToCsvFile(List<HashMap<String, String>> data, String filePath) throws IOException {
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath));
+
+        if (data.size() == 0) {
+            return;
+        }
+
+        List<String> keys = data.get(0).keySet().stream().toList();
+
+        for (String key : keys) {
+            writer.write(key + ",");
+        }
+
+        writer.write("\n");
+
+        for (HashMap<String, String> row : data) {
+            for (String key : keys) {
+                if (row.get(key) == null) {
+                    writer.write(",");
+                    continue;
+                }
+                writer.write(row.get(key) + ",");
+            }
+            writer.write("\n");
+        }
+
+        writer.close();
+    }
+}
diff --git a/validation/src/main/resources/.env.example b/validation/src/main/resources/.env.example
@@ -8,7 +8,7 @@ DATA_FETCHER_START_EPOCH=208
 DATA_FETCHER_END_EPOCH=230
 DATA_FETCHER_SKIP_VALIDATION_DATA=false
 
-# Possible values are: ci, json, db-sync, koios
+# Possible values are: ci, json, db-sync, koios, csv
 SPRING_PROFILES_ACTIVE=ci json
 
 # The following values are required for the dbsync profile

Original file line number	Diff line number	Diff line change
`@@ -15,4 +15,5 @@ public class TreasuryCalculationResult {`
`15`	`15`	`BigInteger totalRewardPot;`
`16`	`16`	`BigInteger treasuryWithdrawals;`
`17`	`17`	`BigInteger unspendableEarnedRewards;`
	`18`	`+ BigInteger unclaimedRefunds;`
`18`	`19`	`}`