Merge pull request #1987 from IntersectMBO/1986-duplicate-pool-stats

1986 - Fix duplicate pool stats

Author: kderme

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -35,6 +35,7 @@ module Test.Cardano.Db.Mock.Config (
   configMetadataEnable,
   configMetadataDisable,
   configMetadataKeys,
+  configPoolStats,
   mkFingerPrint,
   mkMutableDir,
   mkDBSyncEnv,
@@ -357,6 +358,10 @@ configMetadataKeys :: SyncNodeConfig -> SyncNodeConfig
 configMetadataKeys cfg = do
   cfg {dncInsertOptions = (dncInsertOptions cfg) {sioMetadata = MetadataKeys $ 1 :| []}}
 
+configPoolStats :: SyncNodeConfig -> SyncNodeConfig
+configPoolStats cfg = do
+  cfg {dncInsertOptions = (dncInsertOptions cfg) {sioPoolStats = PoolStatsConfig True}}
+
 initCommandLineArgs :: CommandLineArgs
 initCommandLineArgs =
   CommandLineArgs

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs

@@ -127,6 +127,9 @@ unitTests iom knownMigrations =
         , test "rollback stake address cache" Rollback.stakeAddressRollback
         , test "rollback change order of txs" Rollback.rollbackChangeTxOrder
         , test "rollback full tx" Rollback.rollbackFullTx
+        , test "basic pool stats functionality" Rollback.poolStatBasicTest
+        , test "pool stat rollback no duplicates" Rollback.poolStatRollbackNoDuplicates
+        , test "pool stat rollback general" Rollback.poolStatRollbackGeneral
         ]
     , testGroup
         "different configs"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Rollback.hs

@@ -16,10 +16,13 @@ module Test.Cardano.Db.Mock.Unit.Conway.Rollback (
   rollbackChangeTxOrder,
   rollbackFullTx,
   drepDistrRollback,
+  poolStatBasicTest,
+  poolStatRollbackNoDuplicates,
+  poolStatRollbackGeneral,
 ) where
 
 import qualified Cardano.Db as DB
-import Cardano.DbSync.Era.Shelley.Generic.Util (unCredentialHash)
+import Cardano.DbSync.Era.Shelley.Generic (unCredentialHash)
 import Cardano.Ledger.Coin (Coin (..))
 import Cardano.Ledger.Conway.TxCert (ConwayDelegCert (..), Delegatee (..))
 import Cardano.Mock.ChainSync.Server (IOManager (), addBlock, rollback)
@@ -31,11 +34,11 @@ import Cardano.Mock.Forging.Types (PoolIndex (..), StakeIndex (..), UTxOIndex (.
 import Cardano.Prelude
 import Data.Maybe.Strict (StrictMaybe (..))
 import Ouroboros.Network.Block (blockPoint)
-import Test.Cardano.Db.Mock.Config
-import Test.Cardano.Db.Mock.Examples (mockBlock0, mockBlock1, mockBlock2)
+import Test.Cardano.Db.Mock.Config (claFullMode, configPoolStats, conwayConfigDir, initCommandLineArgs, queryDBSync, startDBSync, stopDBSync, withCustomConfigDropDB, withFullConfigDropDB)
+import Test.Cardano.Db.Mock.Examples
 import Test.Cardano.Db.Mock.UnifiedApi
 import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff, assertEqQuery, assertTxCount)
-import Test.Tasty.HUnit (Assertion ())
+import Test.Tasty.HUnit (Assertion (), assertBool, assertEqual)
 import Prelude (error, head, last)
 
 simpleRollback :: IOManager -> [(Text, Text)] -> Assertion
@@ -373,3 +376,108 @@ drepDistrRollback =
       "DrepDistr for epoch 2 should be re-inserted after replay through boundary"
   where
     testLabel = "conwayDrepDistrRollback"
+
+poolStatBasicTest :: IOManager -> [(Text, Text)] -> Assertion
+poolStatBasicTest =
+  withCustomConfigDropDB args (Just configPoolStats) conwayConfigDir testLabel $
+    \interpreter mockServer dbSync -> do
+      startDBSync dbSync
+
+      -- Test basic pool stats functionality
+      void $ registerAllStakeCreds interpreter mockServer
+      assertBlockNoBackoff dbSync 1
+
+      -- Create some epochs with pool stats
+      void $ forgeAndSubmitBlocks interpreter mockServer 200
+      assertBlockNoBackoff dbSync 201
+
+      poolStatCount <- queryDBSync dbSync DB.queryPoolStatCount
+
+      -- Verify pool stats are created and no duplicates exist
+      duplicateCount <- queryDBSync dbSync DB.queryPoolStatDuplicates
+      assertEqual "Should have no duplicate pool stats" 0 duplicateCount
+      assertBool "Should have some pool stats" (poolStatCount > 0)
+  where
+    args = initCommandLineArgs {claFullMode = False}
+    testLabel = "conwayPoolStatBasicTest"
+
+poolStatRollbackNoDuplicates :: IOManager -> [(Text, Text)] -> Assertion
+poolStatRollbackNoDuplicates =
+  withCustomConfigDropDB args (Just configPoolStats) conwayConfigDir testLabel $
+    \interpreter mockServer dbSync -> do
+      startDBSync dbSync
+
+      -- Simple setup: create some blocks with pool stats
+      void $ registerAllStakeCreds interpreter mockServer
+      void $ forgeAndSubmitBlocks interpreter mockServer 200 -- Fill 2 epochs
+      assertBlockNoBackoff dbSync 201
+
+      -- Create rollback point
+      rollbackBlks <- forgeAndSubmitBlocks interpreter mockServer 50
+      assertBlockNoBackoff dbSync 251
+
+      -- Add more blocks to create additional pool stats
+      void $ forgeAndSubmitBlocks interpreter mockServer 100 -- Fill 1 more epoch
+      assertBlockNoBackoff dbSync 351
+
+      -- Rollback (following exact pattern from bigChain test)
+      atomically $ rollback mockServer (blockPoint $ last rollbackBlks)
+      assertBlockNoBackoff dbSync 351 -- Delayed rollbackExpand commentComment on line R345ResolvedCode has comments. Press enter to view.
+
+      -- Re-sync some blocks
+      void $ forgeAndSubmitBlocks interpreter mockServer 100
+      assertBlockNoBackoff dbSync 351 -- Should stay same due to rollbackExpand commentComment on line R349ResolvedCode has comments. Press enter to view.
+
+      -- The main test: no duplicates after rollback + re-sync
+      duplicateCount <- queryDBSync dbSync DB.queryPoolStatDuplicates
+      assertEqual "Should have no duplicate pool stats after rollback" 0 duplicateCount
+  where
+    args = initCommandLineArgs {claFullMode = False}
+    testLabel = "conwayPoolStatRollbackNoDuplicates"
+
+poolStatRollbackGeneral :: IOManager -> [(Text, Text)] -> Assertion
+poolStatRollbackGeneral =
+  withCustomConfigDropDB args (Just configPoolStats) conwayConfigDir testLabel $
+    \interpreter mockServer dbSync -> do
+      startDBSync dbSync
+
+      -- Create pools and stake to generate pool stats
+      void $ registerAllStakeCreds interpreter mockServer
+      epochBlks1 <- fillEpochs interpreter mockServer 2
+
+      -- Create rollback point
+      rollbackBlks <- forgeAndSubmitBlocks interpreter mockServer 10
+      let totalBeforeRollback = length epochBlks1 + length rollbackBlks + 1
+      assertBlockNoBackoff dbSync totalBeforeRollback
+
+      -- Check initial pool stat count
+      initialCount <- queryDBSync dbSync DB.queryPoolStatCount
+
+      -- Forge more blocks to create additional pool stats
+      epochBlks2 <- fillEpochs interpreter mockServer 1
+      let totalAfterEpoch = totalBeforeRollback + length epochBlks2
+      assertBlockNoBackoff dbSync totalAfterEpoch
+
+      -- Verify pool stats increased
+      afterCount <- queryDBSync dbSync DB.queryPoolStatCount
+      assertBool "Pool stats should have increased" (afterCount > initialCount)
+
+      -- Rollback to previous point
+      atomically $ rollback mockServer (blockPoint $ last rollbackBlks)
+      assertBlockNoBackoff dbSync totalAfterEpoch -- Delayed rollbackExpand commentComment on line R387ResolvedCode has comments. Press enter to view.
+
+      -- Re-sync the same blocks - should not create duplicates
+      epochBlks3 <- fillEpochs interpreter mockServer 1
+      let finalTotal = totalBeforeRollback + length epochBlks3 + 1
+      assertBlockNoBackoff dbSync finalTotal
+      finalCount <- queryDBSync dbSync DB.queryPoolStatCount
+
+      -- Verify count matches and no constraint violations occurred
+      assertEqual "Pool stat count should match after rollback" afterCount finalCount
+
+      -- Also verify no duplicates
+      duplicateCount <- queryDBSync dbSync DB.queryPoolStatDuplicates
+      assertEqual "Should have no duplicate pool stats" 0 duplicateCount
+  where
+    args = initCommandLineArgs {claFullMode = False}
+    testLabel = "conwayPoolStatRollbackGeneral"

cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatBasicTest

@@ -0,0 +1 @@
+[12,16,18,21,24,30,31,32,33,40,41,42,43,47,52,60,62,70,80,84,86,92,98,100,106,109,110,111,112,127,134,138,146,149,154,166,168,178,183,188,193,194,198,200,202,220,222,223,224,225,231,239,242,247,261,282,283,288,289,301,302,303,308,313,315,316,320,331,334,344,345,363,364,368,369,375,377,381,389,394,407,418,422,425,430,437,438,439,440,447,450,453,454,456,458,461,467,492,499,507,516,524,538,541,544,546,550,567,573,576,577,579,580,586,589,595,597,603,605,609,616,618,619,623,624,634,636,643,644,659,664,665,672,678,692,705,711,712,719,726,730,739,740,743,747,749,751,754,759,762,763,765,767,773,777,786,788,789,794,801,806,807,829,830,832,849,851,853,869,871,874,875,878,882,888,893,895,896,898,899,903,906,908,911,912,913,922,930,932,938,941,944,950,960,963,966,968,972,977,985,986]

cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackGeneral

@@ -0,0 +1 @@
+[12,16,18,21,24,30,31,32,33,40,41,42,43,47,52,60,62,70,80,84,86,92,98,100,106,109,110,111,112,127,134,138,146,149,154,166,168,178,183,188,193,194,198,200,202,220,222,223,224,225,231,239,242,247,261,282,283,288,289,301,302,303,308,313,315,316,320,331,334,344,345,363,364,368,369,375,377,381,389,394,407,418,422,425,430,437,438,439,440,447,450,453,454,456,458,461,467,492,499,507,516,524,538,541,544,546,550,567,573,576,577,579,580,586,589,595,597,603,605,609,616,618,619,623,624,634,636,643,644,659,664,665,672,678,692,705,711,712,719,726,730,739,740,743,747,749,751,754,759,762,763,765,767,773,777,786,788,789,794,801,806,807,829,830,832,849,851,853,869,871,874,875,878,882,888,893,895,896,898,899,903,906,908,911,912,913,922,930,932,938,941,944,950,960,963,966,968,972,977,985,986,988,990,991,994,997,1002,1034,1035,1036,1039,1041,1051,1059,1061,1063,1068,1076,1081,1082,1091,1101,1102,1104,1106,1123,1126,1138,1141,1143,1144,1149,1162,1167,1172,1177,1181,1184,1185,1188,1190,1203,1205,1214,1215,1221,1233,1234,1245,1249,1250,1251,1261,1265,1266,1267,1272,1281,1283,1289,1294,1300,1304,1307,1308,1310,1314,1315,1319,1325,1334,1350,1353,1359,1362,1370,1371,1373,1375,1381,1399,1404,1415,1416,1419,1420,1425,1426,1435,1436,1437,1441,1444,1448,1458,1461,1462,1467,1470,1479,1486,1489,1494,1496,1497,1513,1519,1528,1529,1538,1549,1551,1553,1555,1567,1580,1583,1595,1601,1603,1613,1614,1616,1625,1637,1638,1639,1640,1643,1653,1654,1655,1658,1667,1672,1674,1683,1692,1700,1706,1709,1712,1714,1715,1717,1726,1733,1750,1756,1758,1759,1771,1778,1781,1783,1789,1797,1800,1801,1816,1823,1828,1831,1840,1847,1852,1858,1872,1873,1874,1876,1893,1894,1895,1900,1905,1912,1918,1923,1930,1940,1944,1949,1952,1953,1958,1965,1966,1968,1972,1974,1975,1994,2005]

cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackNoDuplicates

@@ -0,0 +1 @@
+[12,16,18,21,24,30,31,32,33,40,41,42,43,47,52,60,62,70,80,84,86,92,98,100,106,109,110,111,112,127,134,138,146,149,154,166,168,178,183,188,193,194,198,200,202,220,222,223,224,225,231,239,242,247,261,282,283,288,289,301,302,303,308,313,315,316,320,331,334,344,345,363,364,368,369,375,377,381,389,394,407,418,422,425,430,437,438,439,440,447,450,453,454,456,458,461,467,492,499,507,516,524,538,541,544,546,550,567,573,576,577,579,580,586,589,595,597,603,605,609,616,618,619,623,624,634,636,643,644,659,664,665,672,678,692,705,711,712,719,726,730,739,740,743,747,749,751,754,759,762,763,765,767,773,777,786,788,789,794,801,806,807,829,830,832,849,851,853,869,871,874,875,878,882,888,893,895,896,898,899,903,906,908,911,912,913,922,930,932,938,941,944,950,960,963,966,968,972,977,985,986,988,990,991,994,997,1002,1034,1035,1036,1039,1041,1051,1059,1061,1063,1068,1076,1081,1082,1091,1101,1102,1104,1106,1123,1126,1138,1141,1143,1144,1149,1162,1167,1172,1177,1181,1184,1185,1188,1190,1203,1205,1214,1215,1221,1233,1234,1245,1249,1250,1251,1261,1265,1266,1267,1272,1281,1283,1289,1294,1300,1304,1307,1308,1310,1314,1315,1319,1325,1334,1350,1353,1359,1362,1370,1371,1373,1375,1381,1399,1404,1415,1416,1419,1420,1425,1426,1435,1436,1437,1441,1444,1448,1458,1461,1462,1467,1470,1479,1486,1489,1494,1496,1497,1513,1519,1528,1529,1538,1549,1551,1553,1555,1567,1580,1583,1595,1601,1603,1613,1614,1616,1625,1637,1638,1639,1640,1643,1653,1654,1655,1658,1667,1672,1674,1683,1692,1700,1706,1709,1712,1714,1715,1717,1726,1733,1750,1756,1758,1759,1771,1778,1781,1783,1789,1797,1800,1801,1816,1823,1828,1831,1840,1847,1852,1858,1872,1873,1874,1876,1893,1894,1895,1900,1905,1912,1918,1923,1930,1940,1944,1949,1952,1953,1958,1965,1966,1968,1972,1974,1975,1994,2005,2011,2020,2028,2037,2038,2043,2046,2050,2051,2058,2059,2062,2063,2064,2074,2077,2078,2082,2085,2094,2104,2110,2112,2121,2122,2161,2167,2169,2172,2175,2180,2184,2186,2192,2197,2205,2210,2215,2224,2225,2233,2234,2236,2239,2240,2243,2257,2266,2273,2285,2294,2299,2302,2305,2307,2310,2315]

cardano-db/src/Cardano/Db/Statement/Pool.hs

@@ -208,6 +208,53 @@ insertBulkPoolStat :: [SCP.PoolStat] -> DbM ()
 insertBulkPoolStat poolStats =
   runSession mkDbCallStack $ HsqlSes.statement poolStats insertBulkPoolStatStmt
 
+--------------------------------------------------------------------------------
+queryPoolStatCountStmt :: HsqlStmt.Statement () Int64
+queryPoolStatCountStmt =
+  HsqlStmt.Statement sql encoder decoder True
+  where
+    poolStatTable = tableName (Proxy @SCP.PoolStat)
+
+    sql =
+      TextEnc.encodeUtf8 $
+        Text.concat
+          [ "SELECT COUNT(*)::bigint FROM "
+          , poolStatTable
+          ]
+
+    encoder = mempty
+    decoder = HsqlD.singleRow (HsqlD.column $ HsqlD.nonNullable HsqlD.int8)
+
+queryPoolStatCount :: DbM Int64
+queryPoolStatCount =
+  runSession mkDbCallStack $ HsqlSes.statement () queryPoolStatCountStmt
+
+--------------------------------------------------------------------------------
+queryPoolStatDuplicatesStmt :: HsqlStmt.Statement () Int64
+queryPoolStatDuplicatesStmt =
+  HsqlStmt.Statement sql encoder decoder True
+  where
+    poolStatTable = tableName (Proxy @SCP.PoolStat)
+
+    sql =
+      TextEnc.encodeUtf8 $
+        Text.concat
+          [ "SELECT COUNT(*)::bigint FROM ("
+          , "  SELECT pool_hash_id, epoch_no"
+          , "  FROM "
+          , poolStatTable
+          , "  GROUP BY pool_hash_id, epoch_no"
+          , "  HAVING COUNT(*) > 1"
+          , ") AS duplicates"
+          ]
+
+    encoder = mempty
+    decoder = HsqlD.singleRow (HsqlD.column $ HsqlD.nonNullable HsqlD.int8)
+
+queryPoolStatDuplicates :: DbM Int64
+queryPoolStatDuplicates =
+  runSession mkDbCallStack $ HsqlSes.statement () queryPoolStatDuplicatesStmt
+
 --------------------------------------------------------------------------------
 -- PoolOwner
 --------------------------------------------------------------------------------

schema/migration-2-0045-20250708.sql

@@ -0,0 +1,23 @@
+CREATE FUNCTION migrate() RETURNS void AS $$
+DECLARE
+  next_version int ;
+BEGIN
+  SELECT stage_two + 1 INTO next_version FROM schema_version ;
+  IF next_version = 45 THEN
+
+    -- Remove duplicates
+    DELETE FROM pool_stat
+    WHERE id NOT IN (
+      SELECT DISTINCT ON (pool_hash_id, epoch_no) id
+      FROM pool_stat
+      ORDER BY pool_hash_id, epoch_no, id
+    );
+
+    UPDATE schema_version SET stage_two = next_version ;
+    RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ;
+  END IF ;
+END ;
+$$ LANGUAGE plpgsql ;
+
+SELECT migrate() ;
+DROP FUNCTION migrate() ;