bulk consummed txout

Author: Cmdv

.github/workflows/check-fourmolu.yml

@@ -13,9 +13,9 @@ jobs:
         shell: bash
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Run fourmolu
-        uses: haskell-actions/run-fourmolu@v9
+        uses: haskell-actions/run-fourmolu@v11
         with:
-          version: "0.10.1.0"
+          version: "0.17.0.0"

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Revision history for cardano-db-sync
 
+## 13.7.0.0
+
+### Summary
+- Complete migration from Persistent ORM to Hasql for direct PostgreSQL access.
+
+### Performance Improvements
+- 3-4x faster epoch processing: ~30min → ~8min per epoch
+- Improved cache efficiency: Stake address hit rates increased from 57-79% to 89-99%
+- Reduced memory overhead: Eliminated ORM abstraction layer
+
+### Compatibility
+- PostgreSQL schema remains unchanged
+- Existing database instances compatible without migration
+
 ## 13.6.0.5
 - Fix offchain data so it supports files up to 3MB [#1928]
 - Upgrade to PostgreSQL 17

cabal.project

@@ -10,8 +10,8 @@ repository cardano-haskell-packages
     d4a35cd3121aa00d18544bb0ac01c3e1691d618f462c46129271bccf39f7e8ee
 
 index-state:
-  , hackage.haskell.org 2025-05-23T06:30:40Z
-  , cardano-haskell-packages 2025-05-16T20:03:45Z
+  , hackage.haskell.org 2025-06-26T20:35:31Z
+  , cardano-haskell-packages 2025-06-25T13:51:34Z
 
 packages:
   cardano-db
@@ -85,16 +85,9 @@ if impl (ghc >= 9.12)
     -- https://github.com/kapralVV/Unique/issues/11
     , Unique:hashable
 
-    -- https://github.com/Gabriella439/Haskell-Pipes-Safe-Library/pull/70
-    , pipes-safe:base
-
     -- https://github.com/haskellari/postgresql-simple/issues/152
     , postgresql-simple:base
     , postgresql-simple:template-haskell
-    
-    -- https://github.com/haskell-hvr/int-cast/issues/10
-    , int-cast:base
-
 -- The two following one-liners will cut off / restore the remainder of this file (for nix-shell users):
 -- when using the "cabal" wrapper script provided by nix-shell.
 -- --------------------------- 8< --------------------------

cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway/Scenarios.hs

@@ -5,6 +5,8 @@
 
 #if __GLASGOW_HASKELL__ >= 908
 {-# OPTIONS_GHC -Wno-x-partial #-}
+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
+{-# HLINT ignore "Use zipWith" #-}
 #endif
 
 module Cardano.Mock.Forging.Tx.Conway.Scenarios (

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -62,20 +62,6 @@ module Test.Cardano.Db.Mock.Config (
   txOutVariantTypeFromConfig,
 ) where
 
-import Cardano.Api (NetworkMagic (..))
-import qualified Cardano.Db as DB
-import Cardano.DbSync
-import Cardano.DbSync.Config
-import Cardano.DbSync.Config.Cardano
-import Cardano.DbSync.Config.Types
-import Cardano.DbSync.Error (runOrThrowIO)
-import Cardano.DbSync.Types (CardanoBlock, MetricSetters (..))
-import Cardano.Mock.ChainSync.Server
-import Cardano.Mock.Forging.Interpreter
-import Cardano.Node.Protocol.Shelley (readLeaderCredentials)
-import Cardano.Node.Types (ProtocolFilepaths (..))
-import Cardano.Prelude (NonEmpty ((:|)), panic, stderr, textShow, throwIO)
-import Cardano.SMASH.Server.PoolDataLayer
 import Control.Concurrent.Async (Async, async, cancel, poll)
 import Control.Concurrent.STM (atomically)
 import Control.Concurrent.STM.TMVar (
@@ -92,6 +78,11 @@ import Control.Monad.Logger (NoLoggingT)
 import Control.Monad.Trans.Except.Extra (runExceptT)
 import Control.Tracer (nullTracer)
 import Data.Text (Text)
+import System.Directory (createDirectoryIfMissing, removePathForcibly)
+import System.FilePath.Posix (takeDirectory, (</>))
+import System.IO.Silently (hSilence)
+
+import Cardano.Api (NetworkMagic (..))
 import Ouroboros.Consensus.Block.Forging
 import Ouroboros.Consensus.Byron.Ledger.Mempool ()
 import Ouroboros.Consensus.Config (TopLevelConfig)
@@ -100,9 +91,20 @@ import qualified Ouroboros.Consensus.Node.ProtocolInfo as Consensus
 import Ouroboros.Consensus.Shelley.Eras (StandardCrypto)
 import Ouroboros.Consensus.Shelley.Ledger.Mempool ()
 import Ouroboros.Consensus.Shelley.Node (ShelleyLeaderCredentials)
-import System.Directory (createDirectoryIfMissing, removePathForcibly)
-import System.FilePath.Posix (takeDirectory, (</>))
-import System.IO.Silently (hSilence)
+
+import qualified Cardano.Db as DB
+import Cardano.DbSync
+import Cardano.DbSync.Config
+import Cardano.DbSync.Config.Cardano
+import Cardano.DbSync.Config.Types
+import Cardano.DbSync.Error (runOrThrowIO)
+import Cardano.DbSync.Types (CardanoBlock, MetricSetters (..))
+import Cardano.Mock.ChainSync.Server
+import Cardano.Mock.Forging.Interpreter
+import Cardano.Node.Protocol.Shelley (readLeaderCredentials)
+import Cardano.Node.Types (ProtocolFilepaths (..))
+import Cardano.Prelude (NonEmpty ((:|)), panic, stderr, textShow, throwIO)
+import Cardano.SMASH.Server.PoolDataLayer
 
 data Config = Config
   { topLevelConfig :: TopLevelConfig CardanoBlock
@@ -239,7 +241,7 @@ getPoolLayer env = do
     Left err -> throwIO $ userError err
     Right setting -> pure setting
 
-  -- Create the Hasql connection pool (using port as pool identifier, similar to your server)
+  -- Create the Hasql connection pool, using port as pool identifier
   pool <- DB.createHasqlConnectionPool [connSetting] 1 -- Pool size of 1 for tests
   pure $
     postgresqlPoolDataLayer

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

@@ -34,7 +34,6 @@ insertConfig = do
           , sioPoolStats = PoolStatsConfig False
           , sioJsonType = JsonTypeDisable
           , sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
-          , sioDbDebug = False
           }
 
   dncInsertOptions cfg @?= expected

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Plutus.hs

@@ -28,17 +28,24 @@ module Test.Cardano.Db.Mock.Unit.Alonzo.Plutus (
   swapMultiAssets,
 ) where
 
+import Control.Monad (void)
+import qualified Data.Map as Map
+import Data.Text (Text)
+import Test.Tasty.HUnit (Assertion)
+
 import qualified Cardano.Crypto.Hash as Crypto
-import Cardano.Db (TxOutVariantType (..))
-import qualified Cardano.Db as DB
-import qualified Cardano.Db.Schema.Variants.TxOutAddress as VA
-import qualified Cardano.Db.Schema.Variants.TxOutCore as VC
-import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
 import Cardano.Ledger.Coin
 import Cardano.Ledger.Mary.Value (MaryValue (..), MultiAsset (..), PolicyID (..))
 import Cardano.Ledger.Plutus.Data (hashData)
 import Cardano.Ledger.SafeHash (extractHash)
 import Cardano.Ledger.Shelley.TxCert
+import Ouroboros.Consensus.Cardano.Block (StandardAlonzo)
+
+import Cardano.Db (TxOutVariantType (..))
+import qualified Cardano.Db as DB
+import qualified Cardano.Db.Schema.Variants.TxOutAddress as VA
+import qualified Cardano.Db.Schema.Variants.TxOutCore as VC
+import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
 import Cardano.Mock.ChainSync.Server (IOManager)
 import Cardano.Mock.Forging.Interpreter (withAlonzoLedgerState)
 import qualified Cardano.Mock.Forging.Tx.Alonzo as Alonzo
@@ -57,10 +64,6 @@ import Cardano.Mock.Forging.Types (
   TxEra (..),
   UTxOIndex (..),
  )
-import Control.Monad (void)
-import qualified Data.Map as Map
-import Data.Text (Text)
-import Ouroboros.Consensus.Cardano.Block (StandardAlonzo)
 import Test.Cardano.Db.Mock.Config (alonzoConfigDir, startDBSync, withFullConfig, withFullConfigDropDb)
 import Test.Cardano.Db.Mock.UnifiedApi (
   fillUntilNextEpoch,
@@ -75,7 +78,6 @@ import Test.Cardano.Db.Mock.Validate (
   assertEqQuery,
   assertScriptCert,
  )
-import Test.Tasty.HUnit (Assertion)
 
 ----------------------------------------------------------------------------------------------------------
 -- Plutus Spend Scripts

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Simple.hs

@@ -5,18 +5,20 @@ module Test.Cardano.Db.Mock.Unit.Alonzo.Simple (
   restartDBSync,
 ) where
 
-import Cardano.Ledger.BaseTypes (BlockNo (BlockNo))
-import Cardano.Mock.ChainSync.Server (IOManager, addBlock)
-import Cardano.Mock.Forging.Interpreter (forgeNext)
 import Control.Concurrent.Class.MonadSTM.Strict (MonadSTM (atomically))
 import Control.Monad (void)
 import Data.Text (Text)
+import Test.Tasty.HUnit (Assertion, assertBool)
+
+import Cardano.Ledger.BaseTypes (BlockNo (BlockNo))
 import Ouroboros.Network.Block (blockNo)
+
+import Cardano.Mock.ChainSync.Server (IOManager, addBlock)
+import Cardano.Mock.Forging.Interpreter (forgeNext)
 import Test.Cardano.Db.Mock.Config (alonzoConfigDir, startDBSync, stopDBSync, withFullConfig, withFullConfigDropDb)
 import Test.Cardano.Db.Mock.Examples (mockBlock0, mockBlock1, mockBlock2)
 import Test.Cardano.Db.Mock.UnifiedApi (forgeNextAndSubmit)
 import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff)
-import Test.Tasty.HUnit (Assertion, assertBool)
 
 forgeBlocks :: IOManager -> [(Text, Text)] -> Assertion
 forgeBlocks = do

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Plutus.hs

@@ -30,11 +30,12 @@ module Test.Cardano.Db.Mock.Unit.Babbage.Plutus (
   swapMultiAssets,
 ) where
 
+import Control.Monad (void)
+import qualified Data.Map as Map
+import Data.Text (Text)
+import Test.Tasty.HUnit (Assertion)
+
 import qualified Cardano.Crypto.Hash as Crypto
-import qualified Cardano.Db as DB
-import qualified Cardano.Db.Schema.Variants.TxOutAddress as VA
-import qualified Cardano.Db.Schema.Variants.TxOutCore as VC
-import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
 import Cardano.Ledger.Coin
 import Cardano.Ledger.Mary.Value (MaryValue (..), MultiAsset (..), PolicyID (..))
 import Cardano.Ledger.Plutus.Data (hashData)
@@ -50,6 +51,13 @@ import Cardano.Mock.Forging.Tx.Alonzo.ScriptsExamples (
   assetNames,
   plutusDataList,
  )
+import Ouroboros.Consensus.Cardano.Block (StandardBabbage)
+import Ouroboros.Network.Block (genesisPoint)
+
+import qualified Cardano.Db as DB
+import qualified Cardano.Db.Schema.Variants.TxOutAddress as VA
+import qualified Cardano.Db.Schema.Variants.TxOutCore as VC
+import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
 import qualified Cardano.Mock.Forging.Tx.Babbage as Babbage
 import Cardano.Mock.Forging.Types (
   MockBlock (..),
@@ -58,11 +66,6 @@ import Cardano.Mock.Forging.Types (
   TxEra (..),
   UTxOIndex (..),
  )
-import Control.Monad (void)
-import qualified Data.Map as Map
-import Data.Text (Text)
-import Ouroboros.Consensus.Cardano.Block (StandardBabbage)
-import Ouroboros.Network.Block (genesisPoint)
 import Test.Cardano.Db.Mock.Config (babbageConfigDir, startDBSync, txOutVariantTypeFromConfig, withFullConfig, withFullConfigDropDb)
 import Test.Cardano.Db.Mock.UnifiedApi (
   fillUntilNextEpoch,
@@ -80,7 +83,6 @@ import Test.Cardano.Db.Mock.Validate (
   assertNonZeroFeesContract,
   assertScriptCert,
  )
-import Test.Tasty.HUnit (Assertion)
 
 ----------------------------------------------------------------------------------------------------------
 -- Plutus Spend Scripts

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/JsonbInSchema.hs

@@ -1,4 +1,3 @@
-{-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE OverloadedStrings #-}
 
 module Test.Cardano.Db.Mock.Unit.Conway.Config.JsonbInSchema (