update logs and correct pool_stat error

Author: Cmdv

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -28,6 +28,8 @@ module Cardano.DbSync.Api (
   getPruneConsume,
   getHasConsumedOrPruneTxOut,
   getSkipTxIn,
+  initEpochStatistics,
+  resetEpochStatistics,
   getPrunes,
   mkSyncEnvFromConfig,
   verifySnapshotPoint,
@@ -52,7 +54,7 @@ import Control.Concurrent.Class.MonadSTM.Strict (
   newTVarIO,
   readTVar,
   readTVarIO,
-  writeTVar,
+  writeTVar, StrictTVar,
  )
 import Control.Monad.Trans.Maybe (MaybeT (..))
 import qualified Data.Strict.Maybe as Strict
@@ -77,7 +79,7 @@ import qualified Ouroboros.Network.Point as Point
 
 import qualified Cardano.Db as DB
 import Cardano.DbSync.Api.Types
-import Cardano.DbSync.Cache.Types (CacheCapacity (..), newEmptyCache, useNoCache)
+import Cardano.DbSync.Cache.Types (CacheCapacity (..), newEmptyCache, useNoCache, initCacheStatistics)
 import Cardano.DbSync.Config.Cardano
 import Cardano.DbSync.Config.Shelley
 import Cardano.DbSync.Config.Types
@@ -93,6 +95,7 @@ import Cardano.DbSync.Ledger.Types (HasLedgerEnv (..), LedgerStateFile (..), Sna
 import Cardano.DbSync.LocalStateQuery
 import Cardano.DbSync.Types
 import Cardano.DbSync.Util
+import qualified Data.Map.Strict as Map
 
 setConsistentLevel :: SyncEnv -> ConsistentLevel -> IO ()
 setConsistentLevel env cst = do
@@ -392,6 +395,7 @@ mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfig
   oarq <- newTBQueueIO 1000
   epochVar <- newTVarIO initCurrentEpochNo
   epochSyncTime <- newTVarIO =<< getCurrentTime
+  epochStatistics <- initEpochStatistics
   ledgerEnvType <-
     case (enpMaybeLedgerStateDir syncNP, hasLedger' syncNodeConfigFromFile) of
       (Just dir, True) ->
@@ -417,6 +421,7 @@ mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfig
       { envDbEnv = dbEnv
       , envBootstrap = bootstrapVar
       , envCache = cache
+      , envEpochStatistics = epochStatistics
       , envConsistentLevel = consistentLevelVar
       , envDbConstraints = dbCNamesVar
       , envCurrentEpochNo = epochVar
@@ -437,6 +442,25 @@ mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfig
     hasLedger' = hasLedger . sioLedger . dncInsertOptions
     isTxOutConsumedBootstrap' = isTxOutConsumedBootstrap . sioTxOut . dncInsertOptions
 
+initEpochStatistics :: MonadIO m => m (StrictTVar IO EpochStatistics)
+initEpochStatistics = do
+  curTime <- liftIO getCurrentTime
+  liftIO $ newTVarIO $ EpochStatistics
+    { elsStartTime = curTime
+    , elsCaches = initCacheStatistics
+    , elsUnicodeNull = Map.empty
+    }
+
+resetEpochStatistics :: MonadIO m => SyncEnv -> m ()
+resetEpochStatistics syncEnv = liftIO $ do
+ curTime <- getCurrentTime
+ let newEpochStatsValue = EpochStatistics
+       { elsStartTime = curTime
+       , elsCaches = initCacheStatistics
+       , elsUnicodeNull = Map.empty
+       }
+ atomically $ writeTVar (envEpochStatistics syncEnv) newEpochStatsValue
+
 -- | 'True' is for in memory points and 'False' for on disk
 getLatestPoints :: SyncEnv -> IO [(CardanoPoint, Bool)]
 getLatestPoints env = do

cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs

@@ -156,12 +156,10 @@ prepareTxOut ::
 prepareTxOut syncEnv (TxIn txIntxId (TxIx index), txOut) = do
   let txHashByteString = Generic.safeHashToByteString $ unTxId txIntxId
   let genTxOut = fromTxOut (fromIntegral index) txOut
-  eTxId <- queryTxIdWithCache cache txIntxId
+  eTxId <- queryTxIdWithCache syncEnv txIntxId
   txId <- case eTxId of
     Left err -> throwError err
     Right tid -> pure tid
-  insertTxOut trce cache iopts (txId, txHashByteString) genTxOut
+  insertTxOut syncEnv iopts (txId, txHashByteString) genTxOut
   where
-    trce = getTrace syncEnv
-    cache = envCache syncEnv
     iopts = soptInsertOptions $ envOptions syncEnv

cardano-db-sync/src/Cardano/DbSync/Api/Types.hs

@@ -1,6 +1,7 @@
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE NoImplicitPrelude #-}
+{-# LANGUAGE OverloadedStrings #-}
 
 module Cardano.DbSync.Api.Types (
   SyncEnv (..),
@@ -10,6 +11,9 @@ module Cardano.DbSync.Api.Types (
   RunMigration,
   ConsistentLevel (..),
   CurrentEpochNo (..),
+  UnicodeNullSource(..),
+  EpochStatistics (..),
+  formatUnicodeNullSource,
 ) where
 
 import Control.Concurrent.Class.MonadSTM.Strict (StrictTVar)
@@ -18,12 +22,13 @@ import qualified Data.Strict.Maybe as Strict
 import Data.Time.Clock (UTCTime)
 import Ouroboros.Consensus.BlockchainTime.WallClock.Types (SystemStart (..))
 import Ouroboros.Network.Magic (NetworkMagic (..))
+import qualified Data.Map.Strict as Map
 
-import Cardano.Prelude (Bool, Eq, IO, Show, Word64)
+import Cardano.Prelude (Bool, Eq, IO, Show, Word64, Ord, Text)
 import Cardano.Slotting.Slot (EpochNo (..))
 
 import qualified Cardano.Db as DB
-import Cardano.DbSync.Cache.Types (CacheStatus)
+import Cardano.DbSync.Cache.Types (CacheStatus, CacheStatistics)
 import Cardano.DbSync.Config.Types (SyncNodeConfig)
 import Cardano.DbSync.Ledger.Types (HasLedgerEnv)
 import Cardano.DbSync.LocalStateQuery (NoLedgerEnv)
@@ -38,6 +43,7 @@ import Cardano.DbSync.Types (
 data SyncEnv = SyncEnv
   { envDbEnv :: !DB.DbEnv
   , envCache :: !CacheStatus
+  , envEpochStatistics :: !(StrictTVar IO EpochStatistics)
   , envConsistentLevel :: !(StrictTVar IO ConsistentLevel)
   , envDbConstraints :: !(StrictTVar IO DB.ManualDbConstraints)
   , envCurrentEpochNo :: !(StrictTVar IO CurrentEpochNo)
@@ -98,3 +104,24 @@ data ConsistentLevel = Consistent | DBAheadOfLedger | Unchecked
 newtype CurrentEpochNo = CurrentEpochNo
   { cenEpochNo :: Strict.Maybe EpochNo
   }
+
+data UnicodeNullSource
+  = InsertDatum
+  | InsertRedeemerData
+  | InsertScript
+  | PrepareTxMetadata
+  deriving (Eq, Ord, Show)
+
+
+formatUnicodeNullSource :: UnicodeNullSource -> Text
+formatUnicodeNullSource source = case source of
+  InsertDatum -> "insertDatum: Column 'value' in table 'datum'"
+  InsertRedeemerData -> "insertRedeemerData: Column 'value' in table 'redeemer'"
+  InsertScript -> "insertScript: Column 'json' in table 'script'"
+  PrepareTxMetadata -> "prepareTxMetadata: Column 'json' in table 'tx_metadata'"
+
+data EpochStatistics = EpochStatistics
+  { elsStartTime :: !UTCTime
+  , elsCaches :: !CacheStatistics
+  , elsUnicodeNull :: !(Map.Map UnicodeNullSource [DB.TxId])
+  }