Skip to content

Commit e84e72f

Browse files
CmdvCmdv
committed
bulk consummed txout
1 parent 45161c6 commit e84e72f

File tree

117 files changed

+1920
-3592
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

117 files changed

+1920
-3592
lines changed

‎CHANGELOG.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
# Revision history for cardano-db-sync
22

3+
## 13.7.0.0
4+
5+
### Summary
6+
- Complete migration from Persistent ORM to Hasql for direct PostgreSQL access.
7+
8+
### Performance Improvements
9+
- 3-4x faster epoch processing: ~30min → ~8min per epoch
10+
- Improved cache efficiency: Stake address hit rates increased from 57-79% to 89-99%
11+
- Reduced memory overhead: Eliminated ORM abstraction layer
12+
13+
### Compatibility
14+
- PostgreSQL schema remains unchanged
15+
- Existing database instances compatible without migration
16+
317
## 13.6.0.5
418
- Fix offchain data so it supports files up to 3MB [#1928]
519
- Upgrade to PostgreSQL 17

‎cabal.project

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ repository cardano-haskell-packages
1010
d4a35cd3121aa00d18544bb0ac01c3e1691d618f462c46129271bccf39f7e8ee
1111

1212
index-state:
13-
, hackage.haskell.org 2025-05-23T06:30:40Z
14-
, cardano-haskell-packages 2025-05-16T20:03:45Z
13+
, hackage.haskell.org 2025-06-26T20:35:31Z
14+
, cardano-haskell-packages 2025-06-25T13:51:34Z
1515

1616
packages:
1717
cardano-db
@@ -70,9 +70,6 @@ package snap-server
7070
-- ---------------------------------------------------------
7171

7272
constraints:
73-
-- esqueleto >= 3.6 has API chamges
74-
, esqueleto ^>= 3.5.11.2
75-
7673
-- New version of `text` exposes a `show` function and in the `node`
7774
-- code,`Data.Text` is being imported unqualified (bad idea IMO) which
7875
-- then clashes with the `show` in `Prelude`.
@@ -84,17 +81,6 @@ if impl (ghc >= 9.12)
8481
allow-newer:
8582
-- https://github.com/kapralVV/Unique/issues/11
8683
, Unique:hashable
87-
88-
-- https://github.com/Gabriella439/Haskell-Pipes-Safe-Library/pull/70
89-
, pipes-safe:base
90-
91-
-- https://github.com/haskellari/postgresql-simple/issues/152
92-
, postgresql-simple:base
93-
, postgresql-simple:template-haskell
94-
95-
-- https://github.com/haskell-hvr/int-cast/issues/10
96-
, int-cast:base
97-
9884
-- The two following one-liners will cut off / restore the remainder of this file (for nix-shell users):
9985
-- when using the "cabal" wrapper script provided by nix-shell.
10086
-- --------------------------- 8< --------------------------

‎cardano-chain-gen/cardano-chain-gen.cabal

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,5 @@ test-suite cardano-chain-gen
196196
, ouroboros-consensus
197197
, ouroboros-consensus-cardano
198198
, ouroboros-network-api
199-
, postgresql-simple
200199
, QuickCheck
201200
, quickcheck-state-machine:no-vendored-treediff

‎cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway/Scenarios.hs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
#if __GLASGOW_HASKELL__ >= 908
77
{-# OPTIONS_GHC -Wno-x-partial #-}
8+
{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
9+
{-# HLINT ignore "Use zipWith" #-}
810
#endif
911

1012
module Cardano.Mock.Forging.Tx.Conway.Scenarios (

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -50,32 +50,18 @@ module Test.Cardano.Db.Mock.Config (
5050
startDBSync,
5151
withDBSyncEnv,
5252
withFullConfig,
53-
withFullConfigDropDb,
54-
withFullConfigDropDbLog,
53+
withFullConfigDropDB,
54+
withFullConfigDropDBLog,
5555
withFullConfigLog,
56-
withCustomConfigDropDbLog,
56+
withCustomConfigDropDBLog,
5757
withCustomConfig,
58-
withCustomConfigDropDb,
58+
withCustomConfigDropDB,
5959
withCustomConfigLog,
6060
withFullConfig',
6161
replaceConfigFile,
6262
txOutVariantTypeFromConfig,
6363
) where
6464

65-
import Cardano.Api (NetworkMagic (..))
66-
import qualified Cardano.Db as DB
67-
import Cardano.DbSync
68-
import Cardano.DbSync.Config
69-
import Cardano.DbSync.Config.Cardano
70-
import Cardano.DbSync.Config.Types
71-
import Cardano.DbSync.Error (runOrThrowIO)
72-
import Cardano.DbSync.Types (CardanoBlock, MetricSetters (..))
73-
import Cardano.Mock.ChainSync.Server
74-
import Cardano.Mock.Forging.Interpreter
75-
import Cardano.Node.Protocol.Shelley (readLeaderCredentials)
76-
import Cardano.Node.Types (ProtocolFilepaths (..))
77-
import Cardano.Prelude (NonEmpty ((:|)), panic, stderr, textShow, throwIO)
78-
import Cardano.SMASH.Server.PoolDataLayer
7965
import Control.Concurrent.Async (Async, async, cancel, poll)
8066
import Control.Concurrent.STM (atomically)
8167
import Control.Concurrent.STM.TMVar (
@@ -92,6 +78,11 @@ import Control.Monad.Logger (NoLoggingT)
9278
import Control.Monad.Trans.Except.Extra (runExceptT)
9379
import Control.Tracer (nullTracer)
9480
import Data.Text (Text)
81+
import System.Directory (createDirectoryIfMissing, removePathForcibly)
82+
import System.FilePath.Posix (takeDirectory, (</>))
83+
import System.IO.Silently (hSilence)
84+
85+
import Cardano.Api (NetworkMagic (..))
9586
import Ouroboros.Consensus.Block.Forging
9687
import Ouroboros.Consensus.Byron.Ledger.Mempool ()
9788
import Ouroboros.Consensus.Config (TopLevelConfig)
@@ -100,9 +91,20 @@ import qualified Ouroboros.Consensus.Node.ProtocolInfo as Consensus
10091
import Ouroboros.Consensus.Shelley.Eras (StandardCrypto)
10192
import Ouroboros.Consensus.Shelley.Ledger.Mempool ()
10293
import Ouroboros.Consensus.Shelley.Node (ShelleyLeaderCredentials)
103-
import System.Directory (createDirectoryIfMissing, removePathForcibly)
104-
import System.FilePath.Posix (takeDirectory, (</>))
105-
import System.IO.Silently (hSilence)
94+
95+
import qualified Cardano.Db as DB
96+
import Cardano.DbSync
97+
import Cardano.DbSync.Config
98+
import Cardano.DbSync.Config.Cardano
99+
import Cardano.DbSync.Config.Types
100+
import Cardano.DbSync.Error (runOrThrowIO)
101+
import Cardano.DbSync.Types (CardanoBlock, MetricSetters (..))
102+
import Cardano.Mock.ChainSync.Server
103+
import Cardano.Mock.Forging.Interpreter
104+
import Cardano.Node.Protocol.Shelley (readLeaderCredentials)
105+
import Cardano.Node.Types (ProtocolFilepaths (..))
106+
import Cardano.Prelude (NonEmpty ((:|)), panic, stderr, textShow, throwIO)
107+
import Cardano.SMASH.Server.PoolDataLayer
106108

107109
data Config = Config
108110
{ topLevelConfig :: TopLevelConfig CardanoBlock
@@ -239,7 +241,7 @@ getPoolLayer env = do
239241
Left err -> throwIO $ userError err
240242
Right setting -> pure setting
241243

242-
-- Create the Hasql connection pool (using port as pool identifier, similar to your server)
244+
-- Create the Hasql connection pool, using port as pool identifier
243245
pool <- DB.createHasqlConnectionPool [connSetting] 1 -- Pool size of 1 for tests
244246
pure $
245247
postgresqlPoolDataLayer
@@ -406,7 +408,7 @@ withFullConfig =
406408
Nothing
407409

408410
-- this function needs to be used where the schema needs to be rebuilt
409-
withFullConfigDropDb ::
411+
withFullConfigDropDB ::
410412
-- | config filepath
411413
FilePath ->
412414
-- | test label
@@ -415,7 +417,7 @@ withFullConfigDropDb ::
415417
IOManager ->
416418
[(Text, Text)] ->
417419
IO a
418-
withFullConfigDropDb =
420+
withFullConfigDropDB =
419421
withFullConfig'
420422
( WithConfigArgs
421423
{ hasFingerprint = True
@@ -426,7 +428,7 @@ withFullConfigDropDb =
426428
initCommandLineArgs
427429
Nothing
428430

429-
withFullConfigDropDbLog ::
431+
withFullConfigDropDBLog ::
430432
-- | config filepath
431433
FilePath ->
432434
-- | test label
@@ -435,7 +437,7 @@ withFullConfigDropDbLog ::
435437
IOManager ->
436438
[(Text, Text)] ->
437439
IO a
438-
withFullConfigDropDbLog =
440+
withFullConfigDropDBLog =
439441
withFullConfig'
440442
( WithConfigArgs
441443
{ hasFingerprint = True
@@ -487,7 +489,7 @@ withCustomConfig =
487489
}
488490
)
489491

490-
withCustomConfigDropDb ::
492+
withCustomConfigDropDB ::
491493
CommandLineArgs ->
492494
-- | custom SyncNodeConfig
493495
Maybe (SyncNodeConfig -> SyncNodeConfig) ->
@@ -499,7 +501,7 @@ withCustomConfigDropDb ::
499501
IOManager ->
500502
[(Text, Text)] ->
501503
IO a
502-
withCustomConfigDropDb =
504+
withCustomConfigDropDB =
503505
withFullConfig'
504506
( WithConfigArgs
505507
{ hasFingerprint = True
@@ -530,7 +532,7 @@ withCustomConfigLog =
530532
}
531533
)
532534

533-
withCustomConfigDropDbLog ::
535+
withCustomConfigDropDBLog ::
534536
CommandLineArgs ->
535537
-- | custom SyncNodeConfig
536538
Maybe (SyncNodeConfig -> SyncNodeConfig) ->
@@ -542,7 +544,7 @@ withCustomConfigDropDbLog ::
542544
IOManager ->
543545
[(Text, Text)] ->
544546
IO a
545-
withCustomConfigDropDbLog =
547+
withCustomConfigDropDBLog =
546548
withFullConfig'
547549
( WithConfigArgs
548550
{ hasFingerprint = True

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ insertConfig = do
3434
, sioPoolStats = PoolStatsConfig False
3535
, sioJsonType = JsonTypeDisable
3636
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
37-
, sioDbDebug = False
3837
}
3938

4039
dncInsertOptions cfg @?= expected

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Plutus.hs

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,24 @@ module Test.Cardano.Db.Mock.Unit.Alonzo.Plutus (
2828
swapMultiAssets,
2929
) where
3030

31+
import Control.Monad (void)
32+
import qualified Data.Map as Map
33+
import Data.Text (Text)
34+
import Test.Tasty.HUnit (Assertion)
35+
3136
import qualified Cardano.Crypto.Hash as Crypto
32-
import Cardano.Db (TxOutVariantType (..))
33-
import qualified Cardano.Db as DB
34-
import qualified Cardano.Db.Schema.Variants.TxOutAddress as VA
35-
import qualified Cardano.Db.Schema.Variants.TxOutCore as VC
36-
import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
3737
import Cardano.Ledger.Coin
3838
import Cardano.Ledger.Mary.Value (MaryValue (..), MultiAsset (..), PolicyID (..))
3939
import Cardano.Ledger.Plutus.Data (hashData)
4040
import Cardano.Ledger.SafeHash (extractHash)
4141
import Cardano.Ledger.Shelley.TxCert
42+
import Ouroboros.Consensus.Cardano.Block (StandardAlonzo)
43+
44+
import Cardano.Db (TxOutVariantType (..))
45+
import qualified Cardano.Db as DB
46+
import qualified Cardano.Db.Schema.Variants.TxOutAddress as VA
47+
import qualified Cardano.Db.Schema.Variants.TxOutCore as VC
48+
import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
4249
import Cardano.Mock.ChainSync.Server (IOManager)
4350
import Cardano.Mock.Forging.Interpreter (withAlonzoLedgerState)
4451
import qualified Cardano.Mock.Forging.Tx.Alonzo as Alonzo
@@ -57,11 +64,7 @@ import Cardano.Mock.Forging.Types (
5764
TxEra (..),
5865
UTxOIndex (..),
5966
)
60-
import Control.Monad (void)
61-
import qualified Data.Map as Map
62-
import Data.Text (Text)
63-
import Ouroboros.Consensus.Cardano.Block (StandardAlonzo)
64-
import Test.Cardano.Db.Mock.Config (alonzoConfigDir, startDBSync, withFullConfig, withFullConfigDropDb)
67+
import Test.Cardano.Db.Mock.Config (alonzoConfigDir, startDBSync, withFullConfig, withFullConfigDropDB)
6568
import Test.Cardano.Db.Mock.UnifiedApi (
6669
fillUntilNextEpoch,
6770
forgeNextAndSubmit,
@@ -75,14 +78,13 @@ import Test.Cardano.Db.Mock.Validate (
7578
assertEqQuery,
7679
assertScriptCert,
7780
)
78-
import Test.Tasty.HUnit (Assertion)
7981

8082
----------------------------------------------------------------------------------------------------------
8183
-- Plutus Spend Scripts
8284
----------------------------------------------------------------------------------------------------------
8385
simpleScript :: IOManager -> [(Text, Text)] -> Assertion
8486
simpleScript =
85-
withFullConfigDropDb alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
87+
withFullConfigDropDB alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
8688
startDBSync dbSync
8789
void $ registerAllStakeCreds interpreter mockServer
8890

@@ -271,7 +273,7 @@ multipleScriptsFailedSameBlock =
271273

272274
registrationScriptTx :: IOManager -> [(Text, Text)] -> Assertion
273275
registrationScriptTx =
274-
withFullConfigDropDb alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
276+
withFullConfigDropDB alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
275277
startDBSync dbSync
276278

277279
void $
@@ -394,7 +396,7 @@ deregistrationsScriptTx'' =
394396

395397
mintMultiAsset :: IOManager -> [(Text, Text)] -> Assertion
396398
mintMultiAsset =
397-
withFullConfigDropDb alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
399+
withFullConfigDropDB alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
398400
startDBSync dbSync
399401
void $ withAlonzoFindLeaderAndSubmitTx interpreter mockServer $ \st -> do
400402
let val0 = MultiAsset $ Map.singleton (PolicyID alwaysMintScriptHash) (Map.singleton (head assetNames) 1)

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Simple.hs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,24 @@ module Test.Cardano.Db.Mock.Unit.Alonzo.Simple (
55
restartDBSync,
66
) where
77

8-
import Cardano.Ledger.BaseTypes (BlockNo (BlockNo))
9-
import Cardano.Mock.ChainSync.Server (IOManager, addBlock)
10-
import Cardano.Mock.Forging.Interpreter (forgeNext)
118
import Control.Concurrent.Class.MonadSTM.Strict (MonadSTM (atomically))
129
import Control.Monad (void)
1310
import Data.Text (Text)
11+
import Test.Tasty.HUnit (Assertion, assertBool)
12+
13+
import Cardano.Ledger.BaseTypes (BlockNo (BlockNo))
1414
import Ouroboros.Network.Block (blockNo)
15-
import Test.Cardano.Db.Mock.Config (alonzoConfigDir, startDBSync, stopDBSync, withFullConfig, withFullConfigDropDb)
15+
16+
import Cardano.Mock.ChainSync.Server (IOManager, addBlock)
17+
import Cardano.Mock.Forging.Interpreter (forgeNext)
18+
import Test.Cardano.Db.Mock.Config (alonzoConfigDir, startDBSync, stopDBSync, withFullConfig, withFullConfigDropDB)
1619
import Test.Cardano.Db.Mock.Examples (mockBlock0, mockBlock1, mockBlock2)
1720
import Test.Cardano.Db.Mock.UnifiedApi (forgeNextAndSubmit)
1821
import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff)
19-
import Test.Tasty.HUnit (Assertion, assertBool)
2022

2123
forgeBlocks :: IOManager -> [(Text, Text)] -> Assertion
2224
forgeBlocks = do
23-
withFullConfigDropDb alonzoConfigDir testLabel $ \interpreter _mockServer _dbSync -> do
25+
withFullConfigDropDB alonzoConfigDir testLabel $ \interpreter _mockServer _dbSync -> do
2426
_block0 <- forgeNext interpreter mockBlock0
2527
_block1 <- forgeNext interpreter mockBlock1
2628
block2 <- forgeNext interpreter mockBlock2

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Tx.hs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import Test.Cardano.Db.Mock.Config (
1818
alonzoConfigDir,
1919
startDBSync,
2020
withFullConfig,
21-
withFullConfigDropDb,
21+
withFullConfigDropDB,
2222
)
2323
import Test.Cardano.Db.Mock.UnifiedApi (
2424
withAlonzoFindLeaderAndSubmit,
@@ -29,7 +29,7 @@ import Test.Tasty.HUnit (Assertion)
2929

3030
addSimpleTx :: IOManager -> [(Text, Text)] -> Assertion
3131
addSimpleTx =
32-
withFullConfigDropDb alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
32+
withFullConfigDropDB alonzoConfigDir testLabel $ \interpreter mockServer dbSync -> do
3333
-- translate the block to a real Cardano block.
3434
void $
3535
withAlonzoFindLeaderAndSubmitTx interpreter mockServer $

0 commit comments

Comments
 (0)