Skip to content

Commit 17b263a

Browse files
sorkikderme
sorki
authored and
kderme
committed
Store CBOR serialized transactions in tx_cbor table
Optional feature that can be enabled via insert options like so: ``` "insert_options": { "tx_cbor": "enable", ``` Testing on preview network, with minimal insert options ``` "insert_options": { "tx_cbor": "enable", "tx_out": { "value": "disable" }, "ledger": "disable", "shelley": { "enable": false }, "multi_asset": { "enable": false }, "metadata": { "enable": false }, "plutus": { "enable": false }, "governance": "disable", "offchain_pool_data": "disable", "json_type": "disable" } ``` results in about `6.2GB` of data ``` csyncdb=> SELECT pg_size_pretty(pg_total_relation_size('"public"."tx_cbor"')); pg_size_pretty ---------------- 6272 MB (1 row) ``` Closes #1701
1 parent 4f1ed7d commit 17b263a

File tree

22 files changed

+139
-29
lines changed

22 files changed

+139
-29
lines changed

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ insertConfig = do
2121
cfg <- mkSyncNodeConfig configDir initCommandLineArgs
2222
let expected =
2323
SyncInsertOptions
24-
{ sioTxOut = TxOutDisable
24+
{ sioTxCBOR = TxCBORConfig False
25+
, sioTxOut = TxOutDisable
2526
, sioLedger = LedgerDisable
2627
, sioShelley = ShelleyDisable
2728
, sioRewards = RewardsConfig True

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/Parse.hs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ insertConfig = do
2121
cfg <- mkSyncNodeConfig configDir initCommandLineArgs
2222
let expected =
2323
SyncInsertOptions
24-
{ sioTxOut = TxOutDisable
24+
{ sioTxCBOR = TxCBORConfig False
25+
, sioTxOut = TxOutDisable
2526
, sioLedger = LedgerDisable
2627
, sioShelley = ShelleyDisable
2728
, sioRewards = RewardsConfig True

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ insertConfig = do
9191
cfg <- mkSyncNodeConfig configDir initCommandLineArgs
9292
let expected =
9393
SyncInsertOptions
94-
{ sioTxOut = TxOutDisable
94+
{ sioTxCBOR = TxCBORConfig False
95+
, sioTxOut = TxOutDisable
9596
, sioLedger = LedgerDisable
9697
, sioShelley = ShelleyDisable
9798
, sioRewards = RewardsConfig True

‎cardano-db-sync/CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
# Revision history for cardano-db-sync
22

3+
## Next
4+
* Added an optional feature to store CBOR serialized transactions in `tx_cbor` table [#1723]
5+
36
## 13.2.0.1
47
* Added a new table `epoch_stake_progress` which indicates when `epoch_stake` is completed
58
* Uses the cache for the computation of `epoch` table when following

‎cardano-db-sync/src/Cardano/DbSync.hs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ extractSyncOptions snp aop snc =
266266
iopts =
267267
InsertOptions
268268
{ ioInOut = isTxOutEnabled'
269+
, ioTxCBOR = isTxCBOREnabled (sioTxCBOR (dncInsertOptions snc))
269270
, ioUseLedger = useLedger
270271
, ioShelley = isShelleyEnabled (sioShelley (dncInsertOptions snc))
271272
, -- Rewards are only disabled on "disable_all" and "only_gov" presets

‎cardano-db-sync/src/Cardano/DbSync/Api.hs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,8 @@ extractInsertOptions cfg =
223223
fullInsertOptions :: SyncInsertOptions
224224
fullInsertOptions =
225225
SyncInsertOptions
226-
{ sioTxOut = TxOutEnable
226+
{ sioTxCBOR = TxCBORConfig False
227+
, sioTxOut = TxOutEnable
227228
, sioLedger = LedgerEnable
228229
, sioShelley = ShelleyEnable
229230
, sioRewards = RewardsConfig True
@@ -239,7 +240,8 @@ fullInsertOptions =
239240
onlyUTxOInsertOptions :: SyncInsertOptions
240241
onlyUTxOInsertOptions =
241242
SyncInsertOptions
242-
{ sioTxOut = TxOutBootstrap (ForceTxIn False)
243+
{ sioTxCBOR = TxCBORConfig False
244+
, sioTxOut = TxOutBootstrap (ForceTxIn False)
243245
, sioLedger = LedgerIgnore
244246
, sioShelley = ShelleyDisable
245247
, sioRewards = RewardsConfig True
@@ -262,7 +264,8 @@ onlyGovInsertOptions =
262264
disableAllInsertOptions :: SyncInsertOptions
263265
disableAllInsertOptions =
264266
SyncInsertOptions
265-
{ sioTxOut = TxOutDisable
267+
{ sioTxCBOR = TxCBORConfig False
268+
, sioTxOut = TxOutDisable
266269
, sioLedger = LedgerDisable
267270
, sioShelley = ShelleyDisable
268271
, sioRewards = RewardsConfig False

‎cardano-db-sync/src/Cardano/DbSync/Api/Types.hs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ data SyncOptions = SyncOptions
7474
deriving (Show)
7575

7676
data InsertOptions = InsertOptions
77-
{ ioInOut :: !Bool
77+
{ ioTxCBOR :: !Bool
78+
, ioInOut :: !Bool
7879
, ioUseLedger :: !Bool
7980
, ioShelley :: !Bool
8081
, ioRewards :: !Bool

‎cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ module Cardano.DbSync.Config.Types (
2323
SyncPreConfig (..),
2424
SyncInsertConfig (..),
2525
SyncInsertOptions (..),
26+
TxCBORConfig (..),
2627
TxOutConfig (..),
2728
ForceTxIn (..),
2829
LedgerInsertConfig (..),
@@ -156,7 +157,8 @@ data SyncInsertConfig
156157
deriving (Eq, Show)
157158

158159
data SyncInsertOptions = SyncInsertOptions
159-
{ sioTxOut :: TxOutConfig
160+
{ sioTxCBOR :: TxCBORConfig
161+
, sioTxOut :: TxOutConfig
160162
, sioLedger :: LedgerInsertConfig
161163
, sioShelley :: ShelleyInsertConfig
162164
, sioRewards :: RewardsConfig
@@ -170,6 +172,11 @@ data SyncInsertOptions = SyncInsertOptions
170172
}
171173
deriving (Eq, Show)
172174

175+
newtype TxCBORConfig = TxCBORConfig
176+
{ isTxCBOREnabled :: Bool
177+
}
178+
deriving (Eq, Show)
179+
173180
data TxOutConfig
174181
= TxOutEnable
175182
| TxOutDisable
@@ -391,7 +398,8 @@ instance ToJSON SyncInsertConfig where
391398
instance FromJSON SyncInsertOptions where
392399
parseJSON = Aeson.withObject "SyncInsertOptions" $ \obj ->
393400
SyncInsertOptions
394-
<$> obj .:? "tx_out" .!= sioTxOut def
401+
<$> obj .:? "tx_cbor" .!= sioTxCBOR def
402+
<*> obj .:? "tx_out" .!= sioTxOut def
395403
<*> obj .:? "ledger" .!= sioLedger def
396404
<*> obj .:? "shelley" .!= sioShelley def
397405
<*> pure (sioRewards def)
@@ -406,7 +414,8 @@ instance FromJSON SyncInsertOptions where
406414
instance ToJSON SyncInsertOptions where
407415
toJSON SyncInsertOptions {..} =
408416
Aeson.object
409-
[ "tx_out" .= sioTxOut
417+
[ "tx_cbor" .= sioTxCBOR
418+
, "tx_out" .= sioTxOut
410419
, "ledger" .= sioLedger
411420
, "shelley" .= sioShelley
412421
, "multi_asset" .= sioMultiAsset
@@ -418,6 +427,15 @@ instance ToJSON SyncInsertOptions where
418427
, "remove_jsonb_from_schema" .= sioRemoveJsonbFromSchema
419428
]
420429

430+
instance ToJSON TxCBORConfig where
431+
toJSON = boolToEnableDisable . isTxCBOREnabled
432+
433+
instance FromJSON TxCBORConfig where
434+
parseJSON = Aeson.withText "tx_cbor" $ \v ->
435+
case enableDisableToBool v of
436+
Just g -> pure (TxCBORConfig g)
437+
Nothing -> fail $ "unexpected tx_cbor: " <> show v
438+
421439
instance ToJSON TxOutConfig where
422440
toJSON cfg =
423441
Aeson.object
@@ -593,7 +611,8 @@ instance Default SyncInsertConfig where
593611
instance Default SyncInsertOptions where
594612
def =
595613
SyncInsertOptions
596-
{ sioTxOut = TxOutEnable
614+
{ sioTxCBOR = TxCBORConfig False
615+
, sioTxOut = TxOutEnable
597616
, sioLedger = LedgerEnable
598617
, sioShelley = ShelleyEnable
599618
, sioRewards = RewardsConfig True

‎cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import qualified Cardano.Crypto as Crypto (serializeCborHash)
2020
import Cardano.Db (DbLovelace (..))
2121
import qualified Cardano.Db as DB
2222
import Cardano.DbSync.Api
23-
import Cardano.DbSync.Api.Types (SyncEnv (..), SyncOptions (..))
23+
import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..))
2424
import Cardano.DbSync.Cache (
2525
insertBlockAndCache,
2626
queryPrevBlockWithCache,
@@ -237,24 +237,37 @@ insertByronTx syncEnv blkId tx blockIndex = do
237237
disInOut <- liftIO $ getDisableInOutState syncEnv
238238
if disInOut
239239
then do
240-
void . lift . DB.insertTx $
241-
DB.Tx
242-
{ DB.txHash = Byron.unTxHash $ Crypto.serializeCborHash (Byron.taTx tx)
243-
, DB.txBlockId = blkId
244-
, DB.txBlockIndex = blockIndex
245-
, DB.txOutSum = DbLovelace 0
246-
, DB.txFee = DbLovelace 0
247-
, DB.txDeposit = Nothing -- Byron does not have deposits/refunds
248-
-- Would be really nice to have a way to get the transaction size
249-
-- without re-serializing it.
250-
, DB.txSize = fromIntegral $ BS.length (serialize' $ Byron.taTx tx)
251-
, DB.txInvalidHereafter = Nothing
252-
, DB.txInvalidBefore = Nothing
253-
, DB.txValidContract = True
254-
, DB.txScriptSize = 0
255-
}
240+
txId <-
241+
lift . DB.insertTx $
242+
DB.Tx
243+
{ DB.txHash = Byron.unTxHash $ Crypto.serializeCborHash (Byron.taTx tx)
244+
, DB.txBlockId = blkId
245+
, DB.txBlockIndex = blockIndex
246+
, DB.txOutSum = DbLovelace 0
247+
, DB.txFee = DbLovelace 0
248+
, DB.txDeposit = Nothing -- Byron does not have deposits/refunds
249+
-- Would be really nice to have a way to get the transaction size
250+
-- without re-serializing it.
251+
, DB.txSize = fromIntegral $ BS.length (serialize' $ Byron.taTx tx)
252+
, DB.txInvalidHereafter = Nothing
253+
, DB.txInvalidBefore = Nothing
254+
, DB.txValidContract = True
255+
, DB.txScriptSize = 0
256+
}
257+
258+
when (ioTxCBOR iopts) $ do
259+
void
260+
. lift
261+
. DB.insertTxCBOR
262+
$ DB.TxCbor
263+
{ DB.txCborTxId = txId
264+
, DB.txCborBytes = serialize' $ Byron.taTx tx
265+
}
266+
256267
pure 0
257268
else insertByronTx' syncEnv blkId tx blockIndex
269+
where
270+
iopts = getInsertOptions syncEnv
258271

259272
insertByronTx' ::
260273
(MonadBaseControl IO m, MonadIO m) =>
@@ -284,6 +297,15 @@ insertByronTx' syncEnv blkId tx blockIndex = do
284297
, DB.txScriptSize = 0
285298
}
286299

300+
when (ioTxCBOR iopts) $ do
301+
void
302+
. lift
303+
. DB.insertTxCBOR
304+
$ DB.TxCbor
305+
{ DB.txCborTxId = txId
306+
, DB.txCborBytes = serialize' $ Byron.taTx tx
307+
}
308+
287309
-- Insert outputs for a transaction before inputs in case the inputs for this transaction
288310
-- references the output (not sure this can even happen).
289311
disInOut <- liftIO $ getDisableInOutState syncEnv
@@ -296,6 +318,8 @@ insertByronTx' syncEnv blkId tx blockIndex = do
296318
-- fees are being returned so we can sum them and put them in cache to use when updating epochs
297319
pure $ unDbLovelace $ vfFee valFee
298320
where
321+
iopts = getInsertOptions syncEnv
322+
299323
tracer :: Trace IO Text
300324
tracer = getTrace syncEnv
301325

‎cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Allegra.hs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import Cardano.DbSync.Era.Shelley.Generic.Metadata
1919
import Cardano.DbSync.Era.Shelley.Generic.Script (fromTimelock)
2020
import Cardano.DbSync.Era.Shelley.Generic.Tx.Shelley (
2121
calcWithdrawalSum,
22+
getTxCBOR,
2223
getTxMetadata,
2324
getTxSize,
2425
mkTxCertificates,
@@ -50,6 +51,7 @@ fromAllegraTx (blkIndex, tx) =
5051
Tx
5152
{ txHash = txHashId tx
5253
, txBlockIndex = blkIndex
54+
, txCBOR = getTxCBOR tx
5355
, txSize = getTxSize tx
5456
, txValidContract = True
5557
, txInputs = mkTxIn txBody

0 commit comments

Comments
 (0)