Skip to content

Commit 1a2b326

Browse files
kdermekderme
committed
Add instant_reward and adjust reward table
Move MIR to a separate table Use "generated always as" for earned_epoch Remove primary keys
1 parent 00c2ff3 commit 1a2b326

File tree

13 files changed

+180
-106
lines changed

13 files changed

+180
-106
lines changed

‎cardano-db-sync/src/Cardano/DbSync/Default.hs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import Cardano.DbSync.Era.Cardano.Insert (insertEpochSyncTime)
2323
import Cardano.DbSync.Era.Shelley.Adjust (adjustEpochRewards)
2424
import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
2525
import Cardano.DbSync.Era.Shelley.Insert (insertShelleyBlock, mkAdaPots)
26-
import Cardano.DbSync.Era.Shelley.Insert.Epoch (insertPoolDepositRefunds, insertRewards)
26+
import Cardano.DbSync.Era.Shelley.Insert.Epoch (insertInstantRewards, insertPoolDepositRefunds, insertRewards)
2727
import Cardano.DbSync.Era.Shelley.Validate (validateEpochRewards)
2828
import Cardano.DbSync.Error
2929
import Cardano.DbSync.Fix.EpochStake
@@ -293,7 +293,7 @@ insertLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) =
293293
LedgerMirDist rwd -> do
294294
unless (Map.null rwd) $ do
295295
let rewards = Map.toList rwd
296-
insertRewards syncEnv ntw (subFromCurrentEpoch 1) currentEpochNo cache rewards
296+
insertInstantRewards ntw (subFromCurrentEpoch 1) currentEpochNo cache rewards
297297
liftIO . logInfo tracer $ "Inserted " <> show (length rewards) <> " Mir rewards"
298298
LedgerPoolReap en drs ->
299299
unless (Map.null $ Generic.unRewards drs) $ do

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Adjust.hs

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import Cardano.Slotting.Slot (EpochNo (..))
2222
import Control.Monad.Trans.Control (MonadBaseControl)
2323
import qualified Data.Map.Strict as Map
2424
import qualified Data.Set as Set
25-
import qualified Data.Strict.Maybe as Strict
2625
import Database.Esqueleto.Experimental (
2726
SqlBackend,
2827
delete,
@@ -80,17 +79,15 @@ deleteReward ::
8079
ReaderT SqlBackend m ()
8180
deleteReward nw cache epochNo (cred, rwd) = do
8281
mAddrId <- queryStakeAddrWithCache cache DontCacheNew nw cred
83-
eiPoolId <- case Generic.rewardPool rwd of
84-
Strict.Nothing -> pure $ Left $ Db.DbLookupMessage "deleteReward.queryPoolKeyWithCache"
85-
Strict.Just poolHash -> queryPoolKeyWithCache cache DontCacheNew poolHash
82+
eiPoolId <- queryPoolKeyWithCache cache DontCacheNew (Generic.rewardPool rwd)
8683
case (mAddrId, eiPoolId) of
8784
(Right addrId, Right poolId) -> do
8885
delete $ do
8986
rwdDb <- from $ table @Db.Reward
9087
where_ (rwdDb ^. Db.RewardAddrId ==. val addrId)
9188
where_ (rwdDb ^. Db.RewardType ==. val (Generic.rewardSource rwd))
9289
where_ (rwdDb ^. Db.RewardSpendableEpoch ==. val (unEpochNo epochNo))
93-
where_ (rwdDb ^. Db.RewardPoolId ==. val (Just poolId))
90+
where_ (rwdDb ^. Db.RewardPoolId ==. val poolId)
9491
_ -> pure ()
9592

9693
deleteOrphanedRewards :: MonadIO m => EpochNo -> [Db.StakeAddressId] -> ReaderT SqlBackend m ()

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Rewards.hs

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
module Cardano.DbSync.Era.Shelley.Generic.Rewards (
77
Reward (..),
88
Rewards (..),
9+
InstantReward (..),
10+
InstantRewards (..),
911
rewardsCount,
1012
rewardsTotalAda,
1113
) where
@@ -16,24 +18,31 @@ import Cardano.Ledger.Coin (Coin (..))
1618
import Cardano.Prelude
1719
import qualified Data.Map.Strict as Map
1820
import qualified Data.Set as Set
19-
import qualified Data.Strict.Maybe as Strict
2021
import Ouroboros.Consensus.Cardano.CanHardFork ()
2122

2223
data Reward = Reward
2324
{ rewardSource :: !RewardSource
24-
, rewardPool :: !(Strict.Maybe PoolKeyHash)
25+
, rewardPool :: !PoolKeyHash
2526
, rewardAmount :: !Coin
2627
}
2728
deriving (Eq, Ord, Show)
2829

29-
-- The `ledger-specs` code defines a `RewardUpdate` type that is parameterised over
30-
-- Shelley/Allegra/Mary. This is a huge pain in the neck for `db-sync` so we define a
31-
-- generic one instead.
3230
newtype Rewards = Rewards
3331
{ unRewards :: Map StakeCred (Set Reward)
3432
}
3533
deriving (Eq, Show)
3634

35+
data InstantReward = InstantReward
36+
{ irSource :: !RewardSource
37+
, irAmount :: !Coin
38+
}
39+
deriving (Eq, Ord, Show)
40+
41+
newtype InstantRewards = InstantRewards
42+
{ unIRewards :: Map StakeCred (Set Reward)
43+
}
44+
deriving (Eq, Show)
45+
3746
rewardsCount :: Rewards -> Int
3847
rewardsCount = sum . map Set.size . Map.elems . unRewards
3948

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert/Epoch.hs

Lines changed: 52 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
module Cardano.DbSync.Era.Shelley.Insert.Epoch (
1313
insertRewards,
14+
insertInstantRewards,
1415
insertPoolDepositRefunds,
1516
insertStakeSlice,
1617
sumRewardTotal,
@@ -31,11 +32,9 @@ import qualified Cardano.Ledger.Coin as Shelley
3132
import Cardano.Prelude
3233
import Cardano.Slotting.Slot (EpochNo (..))
3334
import Control.Concurrent.Class.MonadSTM.Strict (readTVarIO)
34-
import Control.Monad.Extra (mapMaybeM)
3535
import Control.Monad.Trans.Control (MonadBaseControl)
3636
import qualified Data.Map.Strict as Map
3737
import qualified Data.Set as Set
38-
import qualified Data.Strict.Maybe as Strict
3938
import Database.Persist.Sql (SqlBackend)
4039

4140
{- HLINT ignore "Use readTVarIO" -}
@@ -117,44 +116,70 @@ insertRewards syncEnv nw earnedEpoch spendableEpoch cache rewardsChunk = do
117116
ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.Reward]
118117
mkRewards (saddr, rset) = do
119118
saId <- lift $ queryOrInsertStakeAddress cache CacheNew nw saddr
120-
mapMaybeM (prepareReward saId) (Set.toList rset)
119+
mapM (prepareReward saId) (Set.toList rset)
121120

122-
-- For rewards with a null pool, the reward unique key doesn't work.
123-
-- So we need to manually check that it's not already in the db.
124-
-- This can happen on rollbacks.
125121
prepareReward ::
126122
(MonadBaseControl IO m, MonadIO m) =>
127123
DB.StakeAddressId ->
128124
Generic.Reward ->
129-
ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.Reward)
125+
ExceptT SyncNodeError (ReaderT SqlBackend m) DB.Reward
130126
prepareReward saId rwd = do
131-
mPool <- queryPool (Generic.rewardPool rwd)
132-
let rwdDb =
133-
DB.Reward
134-
{ DB.rewardAddrId = saId
135-
, DB.rewardType = Generic.rewardSource rwd
136-
, DB.rewardAmount = Generic.coinToDbLovelace (Generic.rewardAmount rwd)
137-
, DB.rewardEarnedEpoch = unEpochNo earnedEpoch
138-
, DB.rewardSpendableEpoch = unEpochNo spendableEpoch
139-
, DB.rewardPoolId = mPool
140-
}
141-
case DB.rewardPoolId rwdDb of
142-
Just _ -> pure $ Just rwdDb
143-
Nothing -> do
144-
exists <- lift $ DB.queryNullPoolRewardExists rwdDb
145-
if exists then pure Nothing else pure (Just rwdDb)
127+
poolId <- queryPool (Generic.rewardPool rwd)
128+
pure $
129+
DB.Reward
130+
{ DB.rewardAddrId = saId
131+
, DB.rewardType = Generic.rewardSource rwd
132+
, DB.rewardAmount = Generic.coinToDbLovelace (Generic.rewardAmount rwd)
133+
, DB.rewardEarnedEpoch = unEpochNo earnedEpoch
134+
, DB.rewardSpendableEpoch = unEpochNo spendableEpoch
135+
, DB.rewardPoolId = poolId
136+
}
146137

147138
queryPool ::
148139
(MonadBaseControl IO m, MonadIO m) =>
149-
Strict.Maybe PoolKeyHash ->
150-
ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.PoolHashId)
151-
queryPool Strict.Nothing = pure Nothing
152-
queryPool (Strict.Just poolHash) =
153-
Just <$> lift (queryPoolKeyOrInsert "insertRewards" trce cache CacheNew (ioShelley iopts) poolHash)
140+
PoolKeyHash ->
141+
ExceptT SyncNodeError (ReaderT SqlBackend m) DB.PoolHashId
142+
queryPool poolHash =
143+
lift (queryPoolKeyOrInsert "insertRewards" trce cache CacheNew (ioShelley iopts) poolHash)
154144

155145
trce = getTrace syncEnv
156146
iopts = getInsertOptions syncEnv
157147

148+
insertInstantRewards ::
149+
(MonadBaseControl IO m, MonadIO m) =>
150+
Network ->
151+
EpochNo ->
152+
EpochNo ->
153+
Cache ->
154+
[(StakeCred, Set Generic.InstantReward)] ->
155+
ExceptT SyncNodeError (ReaderT SqlBackend m) ()
156+
insertInstantRewards nw earnedEpoch spendableEpoch cache rewardsChunk = do
157+
dbRewards <- concatMapM mkRewards rewardsChunk
158+
let chunckDbRewards = splittRecordsEvery 100000 dbRewards
159+
-- minimising the bulk inserts into hundred thousand chunks to improve performance
160+
forM_ chunckDbRewards $ \rws -> lift $ DB.insertManyInstantRewards rws
161+
where
162+
mkRewards ::
163+
(MonadBaseControl IO m, MonadIO m) =>
164+
(StakeCred, Set Generic.InstantReward) ->
165+
ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.InstantReward]
166+
mkRewards (saddr, rset) = do
167+
saId <- lift $ queryOrInsertStakeAddress cache CacheNew nw saddr
168+
pure $ map (prepareReward saId) (Set.toList rset)
169+
170+
prepareReward ::
171+
DB.StakeAddressId ->
172+
Generic.InstantReward ->
173+
DB.InstantReward
174+
prepareReward saId rwd =
175+
DB.InstantReward
176+
{ DB.instantRewardAddrId = saId
177+
, DB.instantRewardType = Generic.irSource rwd
178+
, DB.instantRewardAmount = Generic.coinToDbLovelace (Generic.irAmount rwd)
179+
, DB.instantRewardEarnedEpoch = unEpochNo earnedEpoch
180+
, DB.instantRewardSpendableEpoch = unEpochNo spendableEpoch
181+
}
182+
158183
splittRecordsEvery :: Int -> [a] -> [[a]]
159184
splittRecordsEvery val = go
160185
where

‎cardano-db-sync/src/Cardano/DbSync/Era/Util.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,6 @@ safeDecodeToJson tracer tracePrefix x = do
4747
-- See https://github.com/IntersectMBO/cardano-db-sync/issues/297
4848
if containsUnicodeNul json
4949
then do
50-
liftIO $ logWarning tracer $ tracePrefix <> ": dropped due to a Unicode NUL character."
50+
liftIO $ logWarning tracer $ tracePrefix <> ": dropped due to a Unicode NUL character. " <> json
5151
pure Nothing
5252
else pure $ Just json

‎cardano-db-sync/src/Cardano/DbSync/Ledger/Event.hs

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ import Data.SOP.BasicFunctors
5252
import Data.SOP.Constraint
5353
import Data.SOP.Strict (hcmap, hcollapse)
5454
import qualified Data.Set as Set
55-
import qualified Data.Strict.Maybe as Strict
5655
import Ouroboros.Consensus.Byron.Ledger.Block (ByronBlock)
5756
import Ouroboros.Consensus.Cardano.Block
5857
import Ouroboros.Consensus.HardFork.Combinator.AcrossEras (
@@ -64,7 +63,7 @@ import Ouroboros.Consensus.Shelley.Ledger (ShelleyBlock, ShelleyLedgerEvent (..)
6463
import Ouroboros.Consensus.TypeFamilyWrappers
6564

6665
data LedgerEvent
67-
= LedgerMirDist !(Map StakeCred (Set Generic.Reward))
66+
= LedgerMirDist !(Map StakeCred (Set Generic.InstantReward))
6867
| LedgerPoolReap !EpochNo !Generic.Rewards
6968
| LedgerIncrementalRewards !EpochNo !Generic.Rewards
7069
| LedgerDeltaRewards !EpochNo !Generic.Rewards
@@ -260,30 +259,29 @@ convertPoolDepositRefunds rwds =
260259
convert (kh, coin) =
261260
Generic.Reward
262261
{ Generic.rewardSource = RwdDepositRefund
263-
, Generic.rewardPool = Strict.Just kh
262+
, Generic.rewardPool = kh
264263
, Generic.rewardAmount = coin
265264
}
266265

267266
convertMirRewards ::
268267
Map StakeCred Coin ->
269268
Map StakeCred Coin ->
270-
Map StakeCred (Set Generic.Reward)
269+
Map StakeCred (Set Generic.InstantReward)
271270
convertMirRewards resPay trePay =
272271
Map.unionWith Set.union (convertResPay resPay) (convertTrePay trePay)
273272
where
274-
convertResPay :: Map StakeCred Coin -> Map StakeCred (Set Generic.Reward)
273+
convertResPay :: Map StakeCred Coin -> Map StakeCred (Set Generic.InstantReward)
275274
convertResPay = Map.map (mkPayment RwdReserves)
276275

277-
convertTrePay :: Map StakeCred Coin -> Map StakeCred (Set Generic.Reward)
276+
convertTrePay :: Map StakeCred Coin -> Map StakeCred (Set Generic.InstantReward)
278277
convertTrePay = Map.map (mkPayment RwdTreasury)
279278

280-
mkPayment :: RewardSource -> Coin -> Set Generic.Reward
279+
mkPayment :: RewardSource -> Coin -> Set Generic.InstantReward
281280
mkPayment src coin =
282281
Set.singleton $
283-
Generic.Reward
284-
{ Generic.rewardSource = src
285-
, Generic.rewardPool = Strict.Nothing
286-
, Generic.rewardAmount = coin
282+
Generic.InstantReward
283+
{ Generic.irSource = src
284+
, Generic.irAmount = coin
287285
}
288286

289287
convertPoolRewards ::
@@ -298,7 +296,7 @@ convertPoolRewards rmap =
298296
Generic.Reward
299297
{ Generic.rewardSource = rewardTypeToSource $ Ledger.rewardType sr
300298
, Generic.rewardAmount = Ledger.rewardAmount sr
301-
, Generic.rewardPool = Strict.Just $ Ledger.rewardPool sr
299+
, Generic.rewardPool = Ledger.rewardPool sr
302300
}
303301

304302
--------------------------------------------------------------------------------

‎cardano-db/src/Cardano/Db/Delete.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ deleteBlocksBlockId trce blockId = do
8282
liftIO $
8383
logWarning
8484
trce
85-
"Failed to find ReverseInex. Deletion may take longer."
85+
"Failed to find ReverseIndex. Deletion may take longer."
8686
pure (minIds, False)
8787
Just minIdDB -> do
8888
let minIds' = minIds <> minIdDB

‎cardano-db/src/Cardano/Db/Insert.hs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ module Cardano.Db.Insert (
1818
insertExtraKeyWitness,
1919
insertManyEpochStakes,
2020
insertManyRewards,
21+
insertManyInstantRewards,
2122
insertManyDrepDistr,
2223
insertManyTxIn,
2324
insertMaTxMint,
@@ -199,6 +200,12 @@ insertManyRewards ::
199200
ReaderT SqlBackend m ()
200201
insertManyRewards = insertManyWithManualUnique "Many Rewards"
201202

203+
insertManyInstantRewards ::
204+
(MonadBaseControl IO m, MonadIO m) =>
205+
[InstantReward] ->
206+
ReaderT SqlBackend m ()
207+
insertManyInstantRewards = insertManyCheckUnique "Many Instant Rewards"
208+
202209
insertManyDrepDistr ::
203210
(MonadBaseControl IO m, MonadIO m) =>
204211
[DrepDistr] ->

‎cardano-db/src/Cardano/Db/Migration/Extra/CosnumedTxOut/Schema.hs

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import Cardano.Db.Types (
2626
DbInt65,
2727
DbLovelace,
2828
DbWord64,
29-
RewardSource,
3029
ScriptPurpose,
3130
ScriptType,
3231
SyncState,
@@ -289,24 +288,6 @@ share
289288
txId TxId noreference
290289

291290
-- -----------------------------------------------------------------------------------------------
292-
-- Reward, Stake and Treasury need to be obtained from the ledger state.
293-
294-
-- The reward for each stake address and. This is not a balance, but a reward amount and the
295-
-- epoch in which the reward was earned.
296-
-- This table should never get rolled back.
297-
Reward
298-
addrId StakeAddressId noreference
299-
type RewardSource sqltype=rewardtype
300-
amount DbLovelace sqltype=lovelace
301-
earnedEpoch Word64
302-
spendableEpoch Word64
303-
poolId PoolHashId Maybe noreference
304-
-- Usually NULLables are not allowed in a uniqueness constraint. The semantics of how NULL
305-
-- interacts with those constraints is non-trivial: two NULL values are not considered equal
306-
-- for the purposes of an uniqueness constraint.
307-
-- Use of "!force" attribute on the end of the line disables this check.
308-
UniqueReward addrId type earnedEpoch poolId !force
309-
deriving Show
310291

311292
Withdrawal
312293
addrId StakeAddressId noreference
@@ -740,21 +721,6 @@ schemaDocs =
740721
TxMetadataBytes # "The raw bytes of the payload."
741722
TxMetadataTxId # "The Tx table index of the transaction where this metadata was included."
742723

743-
Reward --^ do
744-
"A table for earned rewards. It includes 5 types of rewards. The rewards are inserted incrementally and\
745-
\ this procedure is finalised when the spendable epoch comes. Before the epoch comes, some entries\
746-
\ may be missing."
747-
RewardAddrId # "The StakeAddress table index for the stake address that earned the reward."
748-
RewardType # "The source of the rewards; pool `member`, pool `leader`, `treasury` or `reserves` payment and pool deposits `refunds`"
749-
RewardAmount # "The reward amount (in Lovelace)."
750-
RewardEarnedEpoch
751-
# "The epoch in which the reward was earned. For `pool` and `leader` rewards spendable in epoch `N`, this will be\
752-
\ `N - 2`, for `treasury` and `reserves` `N - 1` and for `refund` N."
753-
RewardSpendableEpoch # "The epoch in which the reward is actually distributed and can be spent."
754-
RewardPoolId
755-
# "The PoolHash table index for the pool the stake address was delegated to when\
756-
\ the reward is earned or for the pool that there is a deposit refund. Will be NULL for payments from the treasury or the reserves."
757-
758724
Withdrawal --^ do
759725
"A table for withdrawals from a reward account."
760726
WithdrawalAddrId # "The StakeAddress table index for the stake address for which the withdrawal is for."

‎cardano-db/src/Cardano/Db/Query.hs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ module Cardano.Db.Query (
2121
queryCalcEpochEntry,
2222
queryCurrentEpochNo,
2323
queryNormalEpochRewardCount,
24-
queryNullPoolRewardExists,
2524
queryGenesis,
2625
queryGenesisSupply,
2726
queryShelleyGenesisSupply,
@@ -429,17 +428,6 @@ queryNormalEpochRewardCount epochNum = do
429428
pure countRows
430429
pure $ maybe 0 unValue (listToMaybe res)
431430

432-
queryNullPoolRewardExists :: MonadIO m => Reward -> ReaderT SqlBackend m Bool
433-
queryNullPoolRewardExists newRwd = do
434-
res <- select $ do
435-
rwd <- from $ table @Reward
436-
where_ (rwd ^. RewardAddrId ==. val (rewardAddrId newRwd))
437-
where_ (rwd ^. RewardType ==. val (rewardType newRwd))
438-
where_ (rwd ^. RewardEarnedEpoch ==. val (rewardEarnedEpoch newRwd))
439-
limit 1
440-
pure (rwd ^. RewardId)
441-
pure $ not (null res)
442-
443431
queryGenesis :: MonadIO m => ReaderT SqlBackend m (Either LookupFail BlockId)
444432
queryGenesis = do
445433
res <- select $ do

0 commit comments

Comments
 (0)