Skip to content

Commit 106f6a5

Browse files
kdermekderme
committed
Improve Tx hash
Use rolling caches
1 parent 6a467e3 commit 106f6a5

File tree

9 files changed

+53
-72
lines changed

9 files changed

+53
-72
lines changed

‎cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import Cardano.DbSync.Era.Shelley.Generic.Tx.Types (DBPlutusScript)
1616
import qualified Cardano.DbSync.Era.Shelley.Generic.Util as Generic
1717
import Cardano.DbSync.Era.Universal.Insert.Grouped
1818
import Cardano.DbSync.Era.Universal.Insert.Tx (insertTxOut)
19+
import Cardano.DbSync.Era.Util (liftLookupFail)
1920
import Cardano.DbSync.Error
2021
import Cardano.DbSync.Ledger.State
2122
import Cardano.DbSync.Types
@@ -163,7 +164,7 @@ prepareTxOut ::
163164
prepareTxOut syncEnv (TxIn txIntxId (TxIx index), txOut) = do
164165
let txHashByteString = Generic.safeHashToByteString $ unTxId txIntxId
165166
let genTxOut = fromTxOut index txOut
166-
txId <- queryTxIdWithCache cache txIntxId txHashByteString "prepareTxOut"
167+
txId <- liftLookupFail "prepareTxOut" $ queryTxIdWithCache cache txIntxId
167168
insertTxOut trce cache iopts (txId, txHashByteString) genTxOut
168169
where
169170
trce = getTrace syncEnv

‎cardano-db-sync/src/Cardano/DbSync/Cache.hs

Lines changed: 5 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ module Cardano.DbSync.Cache (
2020
queryOrInsertRewardAccount,
2121
insertStakeAddress,
2222
queryStakeAddrWithCache,
23-
resolveInputTxId,
2423
queryTxIdWithCache,
2524
rollbackCache,
2625
tryUpdateCacheTx,
@@ -35,8 +34,6 @@ import Cardano.DbSync.Cache.Epoch (rollbackMapEpochInCache)
3534
import qualified Cardano.DbSync.Cache.FIFO as FIFO
3635
import qualified Cardano.DbSync.Cache.LRU as LRU
3736
import Cardano.DbSync.Cache.Types (CacheAction (..), CacheInternal (..), CacheStatistics (..), CacheStatus (..), initCacheStatistics, isCacheActionUpdate)
38-
import Cardano.DbSync.Era.Shelley.Generic.Tx.Types (TxIn (..))
39-
import qualified Cardano.DbSync.Era.Shelley.Generic.Tx.Types as Generic
4037
import qualified Cardano.DbSync.Era.Shelley.Generic.Util as Generic
4138
import Cardano.DbSync.Era.Shelley.Query
4239
import Cardano.DbSync.Era.Util
@@ -373,39 +370,16 @@ queryTxIdWithCache ::
373370
MonadIO m =>
374371
CacheStatus ->
375372
Ledger.TxId StandardCrypto ->
376-
ByteString ->
377-
Text ->
378-
ExceptT SyncNodeError (ReaderT SqlBackend m) DB.TxId
379-
queryTxIdWithCache cache ledgerTxId txHash errTxt = do
380-
case cache of
381-
NoCache -> liftLookupFail errTxt $ DB.queryTxId txHash
382-
ActiveCache ci -> do
383-
mp <- liftIO $ readTVarIO (cTxIds ci)
384-
case FIFO.lookup ledgerTxId mp of
385-
Just txId -> do
386-
liftIO $ hitTxIds (cStats ci)
387-
pure txId
388-
Nothing -> do
389-
resTxId <- liftLookupFail errTxt $ DB.queryTxId txHash
390-
liftIO $ missTxIds (cStats ci)
391-
liftIO $ atomically $ modifyTVar (cTxIds ci) $ FIFO.insert ledgerTxId resTxId
392-
pure resTxId
393-
394-
resolveInputTxId ::
395-
MonadIO m =>
396-
Generic.TxIn ->
397-
CacheStatus ->
398373
ReaderT SqlBackend m (Either DB.LookupFail DB.TxId)
399-
resolveInputTxId txIn cache = do
400-
let txHash = Generic.txInHash txIn
374+
queryTxIdWithCache cache txIdLedger = do
401375
case cache of
402376
-- Direct database query if no cache.
403377
NoCache -> DB.queryTxId txHash
404378
ActiveCache cacheInternal -> do
405379
-- Read current cache state.
406380
cacheTx <- liftIO $ readTVarIO (cTxIds cacheInternal)
407381

408-
case FIFO.lookup (txInTxId txIn) cacheTx of
382+
case FIFO.lookup txIdLedger cacheTx of
409383
-- Cache hit, return the transaction ID.
410384
Just txId -> do
411385
liftIO $ hitTxIds (cStats cacheInternal)
@@ -417,11 +391,13 @@ resolveInputTxId txIn cache = do
417391
case eTxId of
418392
Right txId -> do
419393
-- Update cache.
420-
liftIO $ atomically $ modifyTVar (cTxIds cacheInternal) $ FIFO.insert (txInTxId txIn) txId
394+
liftIO $ atomically $ modifyTVar (cTxIds cacheInternal) $ FIFO.insert txIdLedger txId
421395
-- Return ID after updating cache.
422396
pure $ Right txId
423397
-- Return lookup failure.
424398
Left _ -> pure $ Left $ DB.DbLookupTxHash txHash
399+
where
400+
txHash = Generic.unTxHash txIdLedger
425401

426402
tryUpdateCacheTx ::
427403
MonadIO m =>

‎cardano-db-sync/src/Cardano/DbSync/Cache/FIFO.hs

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,51 +14,55 @@ module Cardano.DbSync.Cache.FIFO (
1414

1515
import Cardano.Prelude hiding (empty)
1616
import qualified Data.Map.Strict as Map
17-
import Data.Sequence.Strict (StrictSeq)
18-
import qualified Data.Sequence.Strict as Seq
1917

2018
{-
21-
FIFOCache: A First-In-First-Out cache with fixed capacity, using Map.Strict
22-
for key-value storage and Sequence.Strict for maintaining insertion order.
19+
FIFOCache: A First-Ins-First-Outs cache with fixed capacity, using Map.Strict
20+
for key-value storage. It keeps 3 rolling Maps and only the first one is inserted.
21+
When the first one is full, the last Map is removed.
2322
2423
Key operations and their complexities:
25-
- Insertion: O(log n)
26-
- Lookup: O(log n)
27-
- Removal of oldest element: O(1)
24+
- Insertion: O(log mapCapacity)
25+
- Lookup: O(log (mapCapacity * n))
26+
- Removal of oldest elements: O(1)
2827
- Size query: O(1)
2928
-}
3029
data FIFOCache k v = FIFOCache
31-
{ maxCapacity :: !Word64
32-
, cacheMap :: !(Map.Map k v)
33-
, keyOrder :: !(StrictSeq k)
30+
{ mapCapacity :: !Word64
31+
, cacheMap0 :: !(Map.Map k v)
32+
, cacheMap1 :: !(Map.Map k v)
33+
, cacheMap2 :: !(Map.Map k v)
3434
}
3535

3636
empty :: Word64 -> FIFOCache k v
37-
empty capacity = FIFOCache capacity Map.empty Seq.empty
37+
empty capacity = FIFOCache capacity Map.empty Map.empty Map.empty
3838

3939
insert :: Ord k => k -> v -> FIFOCache k v -> FIFOCache k v
4040
insert key value cache@FIFOCache {..}
41-
| Map.size cacheMap >= fromIntegral maxCapacity =
42-
case Seq.lookup 0 keyOrder of
43-
Nothing -> cache -- This should never happen if invariants are maintained
44-
Just oldestKey ->
45-
let !newKeyOrder = key Seq.<| Seq.drop 1 keyOrder
46-
!newMap = Map.insert key value $ Map.delete oldestKey cacheMap
47-
in cache {cacheMap = newMap, keyOrder = newKeyOrder}
41+
| Map.size cacheMap0 >= fromIntegral mapCapacity =
42+
cache
43+
{ cacheMap0 = Map.singleton key value
44+
, cacheMap1 = cacheMap0
45+
, cacheMap2 = cacheMap1
46+
}
4847
| otherwise =
4948
cache
50-
{ cacheMap = Map.insert key value cacheMap
51-
, keyOrder = keyOrder Seq.|> key
49+
{ cacheMap0 = Map.insert key value cacheMap0
5250
}
5351

5452
lookup :: Ord k => k -> FIFOCache k v -> Maybe v
55-
lookup key FIFOCache {..} = Map.lookup key cacheMap
53+
lookup key FIFOCache {..} =
54+
Map.lookup key cacheMap0 <|> Map.lookup key cacheMap1 <|> Map.lookup key cacheMap2
5655

5756
getSize :: FIFOCache k v -> Int
58-
getSize = Map.size . cacheMap
57+
getSize FIFOCache {..} = Map.size cacheMap0 + Map.size cacheMap1 + Map.size cacheMap2
5958

6059
getCapacity :: FIFOCache k v -> Word64
61-
getCapacity = maxCapacity
60+
getCapacity FIFOCache {..} = 3 * mapCapacity
6261

6362
cleanupCache :: FIFOCache k v -> FIFOCache k v
64-
cleanupCache cache = cache {cacheMap = Map.empty, keyOrder = Seq.empty}
63+
cleanupCache cache =
64+
cache
65+
{ cacheMap0 = Map.empty
66+
, cacheMap1 = Map.empty
67+
, cacheMap2 = Map.empty
68+
}

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Shelley.hs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,7 @@ mkTxOut txBody = zipWith fromTxOut [0 ..] $ toList (txBody ^. Core.outputsTxBody
125125
fromTxIn :: Ledger.TxIn StandardCrypto -> TxIn
126126
fromTxIn (Ledger.TxIn (Ledger.TxId txid) (TxIx w64)) =
127127
TxIn
128-
{ txInHash = safeHashToByteString txid
129-
, txInIndex = w64
128+
{ txInIndex = w64
130129
, txInRedeemerIndex = Nothing
131130
, txInTxId = Ledger.TxId txid
132131
}

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Types.hs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,13 @@ module Cardano.DbSync.Era.Shelley.Generic.Tx.Types (
2424
getTxOutDatumHash,
2525
getMaybeDatumHash,
2626
sumTxOutCoin,
27+
toTxHash,
2728
) where
2829

2930
import qualified Cardano.Db as DB
3031
import Cardano.DbSync.Era.Shelley.Generic.Metadata (TxMetadataValue (..))
3132
import Cardano.DbSync.Era.Shelley.Generic.ParamProposal
33+
import Cardano.DbSync.Era.Shelley.Generic.Util
3234
import Cardano.DbSync.Types
3335
import qualified Cardano.Ledger.Address as Ledger
3436
import Cardano.Ledger.Alonzo.Scripts
@@ -96,8 +98,7 @@ data TxWithdrawal = TxWithdrawal
9698
}
9799

98100
data TxIn = TxIn
99-
{ txInHash :: !ByteString
100-
, txInIndex :: !Word64
101+
{ txInIndex :: !Word64
101102
, txInTxId :: !(Ledger.TxId StandardCrypto)
102103
, txInRedeemerIndex :: !(Maybe Word64) -- This only has a meaning for Alonzo.
103104
}
@@ -166,6 +167,9 @@ getMaybeDatumHash (Just hsh) = DatumHash hsh
166167
sumTxOutCoin :: [TxOut] -> Coin
167168
sumTxOutCoin = Coin . sum . map (unCoin . txOutAdaValue)
168169

170+
toTxHash :: TxIn -> ByteString
171+
toTxHash = unTxHash . txInTxId
172+
169173
class AlonzoEraTxBody era => DBScriptPurpose era where
170174
getPurpose :: PlutusPurpose AsIx era -> (DB.ScriptPurpose, Word32)
171175
toAlonzoPurpose :: TxBody era -> PlutusPurpose AsItem era -> Maybe (Either (AlonzoPlutusPurpose AsItem era, Maybe (PlutusPurpose AsIx era)) (ConwayPlutusPurpose AsItem era))

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Query.hs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,16 @@ resolveStakeAddress addr = queryStakeAddress addr renderByteArray
2626

2727
resolveInputTxOutId :: MonadIO m => Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, TxOutId))
2828
resolveInputTxOutId txIn =
29-
queryTxOutId (Generic.txInHash txIn, fromIntegral (Generic.txInIndex txIn))
29+
queryTxOutId (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
3030

3131
resolveInputValue :: MonadIO m => Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, DbLovelace))
3232
resolveInputValue txIn =
33-
queryTxOutValue (Generic.txInHash txIn, fromIntegral (Generic.txInIndex txIn))
33+
queryTxOutValue (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
3434

3535
resolveInputTxOutIdValue :: MonadIO m => Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, TxOutId, DbLovelace))
3636
resolveInputTxOutIdValue txIn =
37-
queryTxOutIdValue (Generic.txInHash txIn, fromIntegral (Generic.txInIndex txIn))
37+
queryTxOutIdValue (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
3838

3939
queryResolveInputCredentials :: MonadIO m => Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (Maybe ByteString, Bool))
4040
queryResolveInputCredentials txIn = do
41-
queryTxOutCredentials (Generic.txInHash txIn, fromIntegral (Generic.txInIndex txIn))
41+
queryTxOutCredentials (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))

‎cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,8 +176,7 @@ resolveGovActionProposal ::
176176
ExceptT SyncNodeError (ReaderT SqlBackend m) DB.GovActionProposalId
177177
resolveGovActionProposal cache gaId = do
178178
let txId = gaidTxId gaId
179-
txHash = Generic.unTxHash txId
180-
gaTxId <- queryTxIdWithCache cache txId txHash "resolveGovActionProposal.queryTxId"
179+
gaTxId <- liftLookupFail "resolveGovActionProposal.queryTxId" $ queryTxIdWithCache cache txId
181180
let (GovActionIx index) = gaidGovActionIx gaId
182181
liftLookupFail "resolveGovActionProposal.queryGovActionProposalId" $
183182
DB.queryGovActionProposalId gaTxId (fromIntegral index) -- TODO: Use Word32?

‎cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import Cardano.Db (DbLovelace (..), minIdsToText)
1919
import qualified Cardano.Db as DB
2020
import Cardano.DbSync.Api
2121
import Cardano.DbSync.Api.Types (SyncEnv (..))
22-
import Cardano.DbSync.Cache (resolveInputTxId)
22+
import Cardano.DbSync.Cache (queryTxIdWithCache)
2323
import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
2424
import Cardano.DbSync.Era.Shelley.Query
2525
import Cardano.DbSync.Era.Util
@@ -156,7 +156,7 @@ resolveTxInputs syncEnv hasConsumed needsValue groupedOutputs txIn =
156156
qres <-
157157
case (hasConsumed, needsValue) of
158158
(_, True) -> fmap convertFoundAll <$> resolveInputTxOutIdValue txIn
159-
(False, _) -> fmap convertnotFound <$> resolveInputTxId txIn (envCache syncEnv)
159+
(False, _) -> fmap convertnotFound <$> queryTxIdWithCache (envCache syncEnv) (Generic.txInTxId txIn)
160160
(True, False) -> fmap convertFoundTxOutId <$> resolveInputTxOutId txIn
161161
case qres of
162162
Right ret -> pure $ Right ret
@@ -214,5 +214,5 @@ resolveInMemory txIn =
214214

215215
matches :: Generic.TxIn -> ExtendedTxOut -> Bool
216216
matches txIn eutxo =
217-
Generic.txInHash txIn == etoTxHash eutxo
217+
Generic.toTxHash txIn == etoTxHash eutxo
218218
&& Generic.txInIndex txIn == DB.txOutIndex (etoTxOut eutxo)

‎cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ import Cardano.DbSync.Era.Universal.Insert.Other (
4141
insertWithdrawals,
4242
)
4343
import Cardano.DbSync.Era.Universal.Insert.Pool (IsPoolMember)
44-
import Cardano.DbSync.Era.Util (safeDecodeToJson)
44+
import Cardano.DbSync.Era.Util (liftLookupFail, safeDecodeToJson)
4545
import Cardano.DbSync.Error
4646
import Cardano.DbSync.Ledger.Types (ApplyResult (..), getGovExpiresAt, lookupDepositsMap)
4747
import Cardano.DbSync.Util
@@ -401,8 +401,7 @@ insertCollateralTxIn ::
401401
ExceptT SyncNodeError (ReaderT SqlBackend m) ()
402402
insertCollateralTxIn syncEnv _tracer txInId txIn = do
403403
let txId = txInTxId txIn
404-
txHash = txInHash txIn
405-
txOutId <- queryTxIdWithCache (envCache syncEnv) txId txHash "insertCollateralTxIn"
404+
txOutId <- liftLookupFail "insertCollateralTxIn" $ queryTxIdWithCache (envCache syncEnv) txId
406405
void
407406
. lift
408407
. DB.insertCollateralTxIn
@@ -421,8 +420,7 @@ insertReferenceTxIn ::
421420
ExceptT SyncNodeError (ReaderT SqlBackend m) ()
422421
insertReferenceTxIn syncEnv _tracer txInId txIn = do
423422
let txId = txInTxId txIn
424-
txHash = txInHash txIn
425-
txOutId <- queryTxIdWithCache (envCache syncEnv) txId txHash "insertReferenceTxIn"
423+
txOutId <- liftLookupFail "insertReferenceTxIn" $ queryTxIdWithCache (envCache syncEnv) txId
426424
void
427425
. lift
428426
. DB.insertReferenceTxIn

0 commit comments

Comments
 (0)