feature(cardano-db-sync): Add new fields to insert options

 * force_tx_in
 * preset

Author: sgillespie

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -117,7 +117,6 @@ data CommandLineArgs = CommandLineArgs
   , claMigrateConsumed :: Bool
   , claPruneTxOut :: Bool
   , claBootstrap :: Bool
-  , claForceTxIn :: Bool
   }
 
 data WithConfigArgs = WithConfigArgs
@@ -275,7 +274,6 @@ mkSyncNodeParams staticDir mutableDir CommandLineArgs {..} = do
       , enpOnlyFix = claOnlyFix
       , enpForceIndexes = claForceIndexes
       , enpHasInOut = True
-      , enpForceTxIn = claForceTxIn
       , enpSnEveryFollowing = 35
       , enpSnEveryLagging = 35
       , enpMaybeRollback = Nothing
@@ -304,7 +302,6 @@ initCommandLineArgs =
     , claMigrateConsumed = False
     , claPruneTxOut = False
     , claBootstrap = False
-    , claForceTxIn = False
     }
 
 emptyMetricsSetters :: MetricSetters

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

@@ -14,24 +14,24 @@ import Prelude ()
 defaultInsertConfig :: Assertion
 defaultInsertConfig = do
   cfg <- mkSyncNodeConfig alonzoConfigDir initCommandLineArgs
-  dncInsertConfig cfg @?= def
+  dncInsertOptions cfg @?= def
 
 insertConfig :: Assertion
 insertConfig = do
   cfg <- mkSyncNodeConfig configDir initCommandLineArgs
   let expected =
-        SyncInsertConfig
-          { spcTxOut = TxOutDisable
-          , spcLedger = LedgerDisable
-          , spcShelley = ShelleyDisable
-          , spcMultiAsset = MultiAssetDisable
-          , spcMetadata = MetadataDisable
-          , spcPlutus = PlutusDisable
-          , spcGovernance = GovernanceConfig False
-          , spcOffchainPoolData = OffchainPoolDataConfig False
-          , spcJsonType = JsonTypeDisable
+        SyncInsertOptions
+          { sioTxOut = TxOutDisable
+          , sioLedger = LedgerDisable
+          , sioShelley = ShelleyDisable
+          , sioMultiAsset = MultiAssetDisable
+          , sioMetadata = MetadataDisable
+          , sioPlutus = PlutusDisable
+          , sioGovernance = GovernanceConfig False
+          , sioOffchainPoolData = OffchainPoolDataConfig False
+          , sioJsonType = JsonTypeDisable
           }
 
-  dncInsertConfig cfg @?= expected
+  dncInsertOptions cfg @?= expected
   where
     configDir = "config-alonzo-insert-options"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/MigrateConsumedPruneTxOut.hs

@@ -16,6 +16,7 @@ module Test.Cardano.Db.Mock.Unit.Babbage.Config.MigrateConsumedPruneTxOut (
 ) where
 
 import qualified Cardano.Db as DB
+import Cardano.DbSync.Config.Types
 import Cardano.Mock.ChainSync.Server (IOManager, addBlock)
 import Cardano.Mock.Forging.Interpreter (forgeNext)
 import qualified Cardano.Mock.Forging.Tx.Babbage as Babbage
@@ -29,6 +30,7 @@ import Test.Cardano.Db.Mock.Config (
   CommandLineArgs (..),
   babbageConfigDir,
   initCommandLineArgs,
+  mkSyncNodeConfig,
   replaceConfigFile,
   startDBSync,
   stopDBSync,
@@ -88,7 +90,6 @@ basicPrune = do
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"
-        , claForceTxIn = True
         }
     testLabel = "configPrune"
 
@@ -117,7 +118,6 @@ pruneWithSimpleRollback = do
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"
-        , claForceTxIn = True
         }
     testLabel = "configPruneSimpleRollback"
 
@@ -148,15 +148,16 @@ pruneWithFullTxRollback = do
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"
-        , claForceTxIn = True
         }
     testLabel = "configPruneOnFullRollback"
 
 -- The tx in the last, 2 x securityParam worth of blocks should not be pruned.
 -- In these tests, 2 x securityParam = 20 blocks.
 pruningShouldKeepSomeTx :: IOManager -> [(Text, Text)] -> Assertion
-pruningShouldKeepSomeTx = do
-  withCustomConfig cmdLineArgs Nothing babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do
+pruningShouldKeepSomeTx ioManager names = do
+  syncNodeConfig <- mkSyncNodeConfig'
+
+  withConfig' syncNodeConfig $ \interpreter mockServer dbSyncEnv -> do
     startDBSync dbSyncEnv
     b1 <- forgeAndSubmitBlocks interpreter mockServer 80
     -- these two blocs + tx will fall withing the last 20 blocks so should not be pruned
@@ -173,6 +174,20 @@ pruningShouldKeepSomeTx = do
     assertTxInCount dbSyncEnv 0
     assertEqQuery dbSyncEnv DB.queryTxOutConsumedCount 0 "Unexpected TxOutConsumedByTxId count after prune"
   where
+    withConfig' cfg f =
+      withCustomConfig cmdLineArgs (Just cfg) babbageConfigDir testLabel f ioManager names
+
+    mkSyncNodeConfig' :: IO SyncNodeConfig
+    mkSyncNodeConfig' = do
+      initCfg <- mkSyncNodeConfig babbageConfigDir cmdLineArgs
+      pure $
+        initCfg
+          { dncInsertOptions =
+              (dncInsertOptions initCfg)
+                { sioTxOut = TxOutPrune (ForceTxIn False)
+                }
+          }
+
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/Parse.hs

@@ -14,24 +14,24 @@ import Prelude ()
 defaultInsertConfig :: Assertion
 defaultInsertConfig = do
   cfg <- mkSyncNodeConfig babbageConfigDir initCommandLineArgs
-  dncInsertConfig cfg @?= def
+  dncInsertOptions cfg @?= def
 
 insertConfig :: Assertion
 insertConfig = do
   cfg <- mkSyncNodeConfig configDir initCommandLineArgs
   let expected =
-        SyncInsertConfig
-          { spcTxOut = TxOutDisable
-          , spcLedger = LedgerDisable
-          , spcShelley = ShelleyDisable
-          , spcMultiAsset = MultiAssetDisable
-          , spcMetadata = MetadataDisable
-          , spcPlutus = PlutusDisable
-          , spcGovernance = GovernanceConfig False
-          , spcOffchainPoolData = OffchainPoolDataConfig False
-          , spcJsonType = JsonTypeDisable
+        SyncInsertOptions
+          { sioTxOut = TxOutDisable
+          , sioLedger = LedgerDisable
+          , sioShelley = ShelleyDisable
+          , sioMultiAsset = MultiAssetDisable
+          , sioMetadata = MetadataDisable
+          , sioPlutus = PlutusDisable
+          , sioGovernance = GovernanceConfig False
+          , sioOffchainPoolData = OffchainPoolDataConfig False
+          , sioJsonType = JsonTypeDisable
           }
 
-  dncInsertConfig cfg @?= expected
+  dncInsertOptions cfg @?= expected
   where
     configDir = "config-babbage-insert-options"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/MigrateConsumedPruneTxOut.hs

@@ -16,7 +16,7 @@ module Test.Cardano.Db.Mock.Unit.Conway.Config.MigrateConsumedPruneTxOut (
 ) where
 
 import qualified Cardano.Db as DB
-import Cardano.DbSync.Config (SyncNodeConfig (..))
+import Cardano.DbSync.Config.Types
 import Cardano.Mock.ChainSync.Server (IOManager (), addBlock)
 import Cardano.Mock.Forging.Interpreter (forgeNext)
 import qualified Cardano.Mock.Forging.Tx.Conway as Conway
@@ -99,7 +99,6 @@ basicPrune = do
     args =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"
-        , claForceTxIn = True
         }
     testLabel = "conwayConfigPrune"
     fullBlockSize b = fromIntegral $ length b + 2
@@ -140,7 +139,6 @@ pruneWithSimpleRollback =
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"
-        , claForceTxIn = True
         }
     testLabel = "conwayConfigPruneSimpleRollback"
     fullBlockSize b = fromIntegral $ length b + 4
@@ -181,14 +179,15 @@ pruneWithFullTxRollback =
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"
-        , claForceTxIn = True
         }
     testLabel = "conwayConfigPruneOnFullRollback"
 
 -- The transactions in the last `2 * securityParam` blocks should not be pruned
 pruningShouldKeepSomeTx :: IOManager -> [(Text, Text)] -> Assertion
-pruningShouldKeepSomeTx =
-  withCustomConfig cmdLineArgs Nothing conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do
+pruningShouldKeepSomeTx ioManager names = do
+  syncNodeConfig <- mkSyncNodeConfig'
+
+  withConfig' syncNodeConfig $ \interpreter mockServer dbSync -> do
     startDBSync dbSync
 
     -- Forge some blocks
@@ -213,6 +212,20 @@ pruningShouldKeepSomeTx =
     assertTxInCount dbSync 0
     assertEqQuery dbSync DB.queryTxOutConsumedCount 0 "Unexpected TxOutConsumedByTxId count after prune"
   where
+    withConfig' cfg f =
+      withCustomConfig cmdLineArgs (Just cfg) conwayConfigDir testLabel f ioManager names
+
+    mkSyncNodeConfig' :: IO SyncNodeConfig
+    mkSyncNodeConfig' = do
+      initCfg <- mkSyncNodeConfig conwayConfigDir cmdLineArgs
+      pure $
+        initCfg
+          { dncInsertOptions =
+              (dncInsertOptions initCfg)
+                { sioTxOut = TxOutPrune (ForceTxIn False)
+                }
+          }
+
     cmdLineArgs =
       initCommandLineArgs
         { claConfigFilename = "test-db-sync-config-prune.json"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs

@@ -1,4 +1,3 @@
-{-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE OverloadedStrings #-}
 
 module Test.Cardano.Db.Mock.Unit.Conway.Config.Parse (
@@ -9,22 +8,15 @@ module Test.Cardano.Db.Mock.Unit.Conway.Config.Parse (
   wrongConwayGenesisHash,
   insertConfig,
   defaultInsertConfig,
-  basicPrune,
 ) where
 
-import qualified Cardano.Db as DB
 import Cardano.DbSync.Config
 import Cardano.DbSync.Config.Types
 import Cardano.DbSync.Error
-import Cardano.Mock.ChainSync.Server (IOManager ())
-import qualified Cardano.Mock.Forging.Tx.Conway as Conway
-import Cardano.Mock.Forging.Types (UTxOIndex (..))
 import Cardano.Prelude hiding (from, isNothing)
 import qualified Data.Aeson as Aeson
 import Data.Default.Class (Default (..))
 import Test.Cardano.Db.Mock.Config
-import Test.Cardano.Db.Mock.UnifiedApi
-import Test.Cardano.Db.Mock.Validate
 import Test.Tasty.HUnit (Assertion (), assertBool, (@?=))
 import Prelude ()
 
@@ -90,64 +82,26 @@ isConwayConfigError = either isConwayConfigError' (const False)
 defaultInsertConfig :: Assertion
 defaultInsertConfig = do
   cfg <- mkSyncNodeConfig configDir initCommandLineArgs
-  dncInsertConfig cfg @?= def
+  dncInsertOptions cfg @?= def
   where
     configDir = "config-conway"
 
 insertConfig :: Assertion
 insertConfig = do
   cfg <- mkSyncNodeConfig configDir initCommandLineArgs
   let expected =
-        SyncInsertConfig
-          { spcTxOut = TxOutDisable
-          , spcLedger = LedgerDisable
-          , spcShelley = ShelleyDisable
-          , spcMultiAsset = MultiAssetDisable
-          , spcMetadata = MetadataDisable
-          , spcPlutus = PlutusDisable
-          , spcGovernance = GovernanceConfig False
-          , spcOffchainPoolData = OffchainPoolDataConfig False
-          , spcJsonType = JsonTypeDisable
+        SyncInsertOptions
+          { sioTxOut = TxOutDisable
+          , sioLedger = LedgerDisable
+          , sioShelley = ShelleyDisable
+          , sioMultiAsset = MultiAssetDisable
+          , sioMetadata = MetadataDisable
+          , sioPlutus = PlutusDisable
+          , sioGovernance = GovernanceConfig False
+          , sioOffchainPoolData = OffchainPoolDataConfig False
+          , sioJsonType = JsonTypeDisable
           }
 
-  dncInsertConfig cfg @?= expected
+  dncInsertOptions cfg @?= expected
   where
     configDir = "config-conway-insert-options"
-
-basicPrune :: IOManager -> [(Text, Text)] -> Assertion
-basicPrune = do
-  withCustomConfigAndDropDB args Nothing cfgDir testLabel $ \interpreter mockServer dbSync -> do
-    startDBSync dbSync
-
-    -- Add some blocks
-    blks <- forgeAndSubmitBlocks interpreter mockServer 50
-
-    -- Add blocks with transactions
-    void $
-      withConwayFindLeaderAndSubmitTx interpreter mockServer $
-        Conway.mkPaymentTx (UTxOIndex 0) (UTxOIndex 1) 10_000 10_000
-    void $
-      withConwayFindLeaderAndSubmitTx interpreter mockServer $
-        Conway.mkPaymentTx (UTxOIndex 1) (UTxOIndex 0) 10_000 10_000
-
-    -- Check tx-out count before pruning
-    assertBlockNoBackoff dbSync (fullBlockSize blks)
-    assertEqQuery dbSync DB.queryTxOutCount 14 "new epoch didn't prune tx_out column that are null"
-
-    blks' <- forgeAndSubmitBlocks interpreter mockServer 48
-    assertBlockNoBackoff dbSync (fullBlockSize $ blks <> blks')
-
-    -- Check that tx_out was pruned
-    assertEqQuery dbSync DB.queryTxOutCount 12 "the pruning didn't work correctly as the tx-out count is incorrect"
-    -- Check unspent tx
-    assertUnspentTx dbSync
-  where
-    args =
-      initCommandLineArgs
-        { claConfigFilename = "test-db-sync-config-prune.json"
-        , claMigrateConsumed = True
-        , claPruneTxOut = True
-        }
-    testLabel = "conwayConfigPrune"
-    fullBlockSize b = fromIntegral $ length b + 2
-    cfgDir = conwayConfigDir

cardano-chain-gen/test/testfiles/config-alonzo-insert-options/test-db-sync-config.json

@@ -112,7 +112,9 @@
     }
   ],
   "insert_options": {
-    "tx_out": "disable",
+    "tx_out": {
+      "value": "disable"
+    },
     "ledger": "disable",
     "shelley": {
       "enable": false,

cardano-chain-gen/test/testfiles/config-babbage-insert-options/test-db-sync-config.json

@@ -112,7 +112,9 @@
     }
   ],
   "insert_options": {
-    "tx_out": "disable",
+    "tx_out": {
+      "value": "disable"
+    },
     "ledger": "disable",
     "shelley": {
       "enable": false,

cardano-chain-gen/test/testfiles/config-conway-insert-options/test-db-sync-config.json

@@ -112,7 +112,9 @@
     }
   ],
   "insert_options": {
-    "tx_out": "disable",
+    "tx_out": {
+      "value": "disable",
+    },
     "ledger": "disable",
     "shelley": {
       "enable": false,

cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-bootstrap.json

@@ -112,6 +112,8 @@
     }
   ],
   "insert_options": {
-    "tx_out": "bootstrap"
+    "tx_out": {
+      "value": "bootstrap"
+    }
   }
 }