fix(cardano-chain-gen): Fix insert options issues

 * Validate state-dir/ledger config
 * Lookup in/out from config
 * Disable rewards in some presets

Author: sgillespie

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

@@ -24,6 +24,7 @@ insertConfig = do
           { sioTxOut = TxOutDisable
           , sioLedger = LedgerDisable
           , sioShelley = ShelleyDisable
+          , sioRewards = RewardsConfig True
           , sioMultiAsset = MultiAssetDisable
           , sioMetadata = MetadataDisable
           , sioPlutus = PlutusDisable

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/Parse.hs

@@ -24,6 +24,7 @@ insertConfig = do
           { sioTxOut = TxOutDisable
           , sioLedger = LedgerDisable
           , sioShelley = ShelleyDisable
+          , sioRewards = RewardsConfig True
           , sioMultiAsset = MultiAssetDisable
           , sioMetadata = MetadataDisable
           , sioPlutus = PlutusDisable

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs

@@ -94,6 +94,7 @@ insertConfig = do
           { sioTxOut = TxOutDisable
           , sioLedger = LedgerDisable
           , sioShelley = ShelleyDisable
+          , sioRewards = RewardsConfig True
           , sioMultiAsset = MultiAssetDisable
           , sioMetadata = MetadataDisable
           , sioPlutus = PlutusDisable

cardano-db-sync/app/cardano-db-sync.hs

@@ -4,6 +4,7 @@
 import Cardano.Db (MigrationDir (..), PGPassSource (PGPassDefaultEnv, PGPassEnv), gitRev)
 import Cardano.DbSync (runDbSyncNode)
 import Cardano.DbSync.Config
+import Cardano.DbSync.Config.Types
 import Cardano.DbSync.Metrics (withMetricSetters)
 import Cardano.Prelude
 import Cardano.Slotting.Slot (SlotNo (..))
@@ -15,6 +16,7 @@ import Options.Applicative (Parser, ParserInfo)
 import qualified Options.Applicative as Opt
 import Paths_cardano_db_sync (version)
 import System.Info (arch, compilerName, compilerVersion, os)
+import Prelude (error)
 
 main :: IO ()
 main = do
@@ -23,26 +25,36 @@ main = do
     CmdVersion -> runVersionCommand
     CmdRun params -> run params
   where
-    -- TODO[sgillespie]: This check needs to come after we parse the config file
-    --
-    -- (Nothing, True) -> error stateDirErrorMsg
-    --
-    -- stateDirErrorMsg :: [Char]
-    -- stateDirErrorMsg =
-    --   "Error: If not using --state-dir then make sure to have --disable-ledger. "
-    --     <> "For more details view https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/syncing-and-rollbacks.md#ledger-state"
-
     knownMigrationsPlain :: [(Text, Text)]
     knownMigrationsPlain = (\x -> (hash x, filepath x)) <$> knownMigrations
 
     run :: SyncNodeParams -> IO ()
     run prms = do
       -- read the config file as early as possible
       syncNodeConfigFromFile <- readSyncNodeConfig (enpConfigFile prms)
+
+      -- Validate state-dir/ledger
+      let maybeLedgerStateDir = enpMaybeLedgerStateDir prms
+          ledgerCfg = sioLedger (dncInsertOptions syncNodeConfigFromFile)
+
+      void $ case (maybeLedgerStateDir, ledgerCfg) of
+        -- It is an error to enable ledger and not specify the state
+        (Nothing, LedgerEnable) -> error stateDirErrorMsg
+        -- Or to ignore ledger and not specify the state
+        (Nothing, LedgerIgnore) -> error stateDirErrorMsg
+        -- Otherwise, it's OK
+        _ -> pure ()
+
       let prometheusPort = dncPrometheusPort syncNodeConfigFromFile
       withMetricSetters prometheusPort $ \metricsSetters ->
         runDbSyncNode metricsSetters knownMigrationsPlain prms syncNodeConfigFromFile
 
+    stateDirErrorMsg :: [Char]
+    stateDirErrorMsg =
+      "Error: If not using --state-dir then make sure to have ledger disabled. "
+        <> "For more details view https://github.com/IntersectMBO/cardano-db-sync/blob"
+        <> "/master/doc/syncing-and-rollbacks.md#ledger-state"
+
 -- -------------------------------------------------------------------------------------------------
 
 opts :: ParserInfo SyncCommand

cardano-db-sync/src/Cardano/DbSync.hs

@@ -253,10 +253,11 @@ extractSyncOptions snp aop snc =
 
     iopts =
       InsertOptions
-        { ioInOut = enpHasInOut snp
+        { ioInOut = isTxOutEnabled'
         , ioUseLedger = useLedger
         , ioShelley = isShelleyEnabled (sioShelley (dncInsertOptions snc))
-        , ioRewards = True
+        , -- Rewards are only disabled on "disable_all" and "only_gov" presets
+          ioRewards = True
         , ioMultiAssets = isMultiAssetEnabled (sioMultiAsset (dncInsertOptions snc))
         , ioMetadata = isMetadataEnabled (sioMetadata (dncInsertOptions snc))
         , ioKeepMetadataNames = maybeKeepMNames
@@ -274,6 +275,7 @@ extractSyncOptions snp aop snc =
     isTxOutConsumed' = isTxOutConsumed . sioTxOut . dncInsertOptions $ snc
     isTxOutPrune' = isTxOutPrune . sioTxOut . dncInsertOptions $ snc
     isTxOutBootstrap' = isTxOutBootstrap . sioTxOut . dncInsertOptions $ snc
+    isTxOutEnabled' = isTxOutEnabled . sioTxOut . dncInsertOptions $ snc
     forceTxIn' = forceTxIn . sioTxOut . dncInsertOptions $ snc
 
 startupReport :: Trace IO Text -> Bool -> SyncNodeParams -> IO ()

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -215,6 +215,7 @@ fullInsertOptions =
     { sioTxOut = TxOutEnable
     , sioLedger = LedgerEnable
     , sioShelley = ShelleyEnable
+    , sioRewards = RewardsConfig True
     , sioMultiAsset = MultiAssetEnable
     , sioMetadata = MetadataEnable
     , sioPlutus = PlutusEnable
@@ -229,6 +230,7 @@ onlyUTxOInsertOptions =
     { sioTxOut = TxOutEnable
     , sioLedger = LedgerIgnore
     , sioShelley = ShelleyDisable
+    , sioRewards = RewardsConfig True
     , sioMultiAsset = MultiAssetDisable
     , sioMetadata = MetadataDisable
     , sioPlutus = PlutusDisable
@@ -248,8 +250,9 @@ disableAllInsertOptions :: SyncInsertOptions
 disableAllInsertOptions =
   SyncInsertOptions
     { sioTxOut = TxOutDisable
-    , sioLedger = LedgerIgnore
+    , sioLedger = LedgerDisable
     , sioShelley = ShelleyDisable
+    , sioRewards = RewardsConfig False
     , sioMultiAsset = MultiAssetDisable
     , sioMetadata = MetadataDisable
     , sioPlutus = PlutusDisable
@@ -408,8 +411,8 @@ mkSyncEnv trce backend connectionString syncOptions protoInfo nw nwMagic systemS
       (Nothing, False) -> NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
       (Just _, False) -> do
         logWarning trce $
-          "Using `--disable-ledger` doesn't require having a --state-dir."
-            <> " For more details view https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#--disable-ledger"
+          "Disabling the ledger doesn't require having a --state-dir."
+            <> " For more details view https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#ledger"
         NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
       -- This won't ever call because we error out this combination at parse time
       (Nothing, True) -> NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart

cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

@@ -26,6 +26,7 @@ module Cardano.DbSync.Config.Types (
   ForceTxIn (..),
   LedgerInsertConfig (..),
   ShelleyInsertConfig (..),
+  RewardsConfig (..),
   MultiAssetConfig (..),
   MetadataConfig (..),
   PlutusConfig (..),
@@ -157,6 +158,7 @@ data SyncInsertOptions = SyncInsertOptions
   { sioTxOut :: TxOutConfig
   , sioLedger :: LedgerInsertConfig
   , sioShelley :: ShelleyInsertConfig
+  , sioRewards :: RewardsConfig
   , sioMultiAsset :: MultiAssetConfig
   , sioMetadata :: MetadataConfig
   , sioPlutus :: PlutusConfig
@@ -190,6 +192,10 @@ data ShelleyInsertConfig
   | ShelleyStakeAddrs (NonEmpty ShortByteString)
   deriving (Eq, Show)
 
+newtype RewardsConfig = RewardsConfig
+  {areRewardsEnabled :: Bool}
+  deriving (Eq, Show)
+
 data MultiAssetConfig
   = MultiAssetEnable
   | MultiAssetDisable
@@ -381,6 +387,7 @@ instance FromJSON SyncInsertOptions where
       <$> obj .:? "tx_out" .!= sioTxOut def
       <*> obj .:? "ledger" .!= sioLedger def
       <*> obj .:? "shelley" .!= sioShelley def
+      <*> pure (sioRewards def)
       <*> obj .:? "multi_asset" .!= sioMultiAsset def
       <*> obj .:? "metadata" .!= sioMetadata def
       <*> obj .:? "plutus" .!= sioPlutus def
@@ -571,6 +578,7 @@ instance Default SyncInsertOptions where
       { sioTxOut = TxOutEnable
       , sioLedger = LedgerEnable
       , sioShelley = ShelleyEnable
+      , sioRewards = RewardsConfig True
       , sioMultiAsset = MultiAssetEnable
       , sioMetadata = MetadataEnable
       , sioPlutus = PlutusEnable

cardano-db-sync/test/Cardano/DbSync/ApiTest.hs

@@ -14,24 +14,16 @@ tests =
   checkParallel $
     Group
       "Cardano.DbSync.Api"
-      [("extractInsertOptions", prop_extractInsertOptions)]
+      [ ("extractInsertOptions", prop_extractInsertOptions)
+      , ("extractInsertOptions rewards", prop_extractInsertOptionsRewards)
+      ]
 
 prop_extractInsertOptions :: Property
 prop_extractInsertOptions = property $ do
   cfg <- forAll Gen.syncPreConfig
 
   let insertOpts = pcInsertConfig cfg
-
-      isSyncInsertConfig =
-        case insertOpts of
-          SyncInsertConfig _ -> True
-          _ -> False
-
-  cover 5 "full" (insertOpts == FullInsertOptions)
-  cover 5 "only utxo" (insertOpts == OnlyUTxOInsertOptions)
-  cover 5 "only gov" (insertOpts == OnlyGovInsertOptions)
-  cover 5 "disable all" (insertOpts == DisableAllInsertOptions)
-  cover 5 "config" isSyncInsertConfig
+  coverInsertCfg insertOpts
 
   case insertOpts of
     FullInsertOptions ->
@@ -44,3 +36,32 @@ prop_extractInsertOptions = property $ do
       extractInsertOptions cfg === disableAllInsertOptions
     SyncInsertConfig cfg' ->
       extractInsertOptions cfg === cfg'
+
+prop_extractInsertOptionsRewards :: Property
+prop_extractInsertOptionsRewards = property $ do
+  cfg <- forAll Gen.syncPreConfig
+
+  let insertOpts = pcInsertConfig cfg
+  coverInsertCfg insertOpts
+
+  let areRewardsEnabled' = areRewardsEnabled $ sioRewards (extractInsertOptions cfg)
+
+  case insertOpts of
+    OnlyGovInsertOptions ->
+      assert $ not areRewardsEnabled'
+    DisableAllInsertOptions ->
+      assert $ not areRewardsEnabled'
+    _ -> assert areRewardsEnabled'
+
+coverInsertCfg :: MonadTest m => SyncInsertConfig -> m ()
+coverInsertCfg insertOpts = do
+  cover 5 "full" (insertOpts == FullInsertOptions)
+  cover 5 "only utxo" (insertOpts == OnlyUTxOInsertOptions)
+  cover 5 "only gov" (insertOpts == OnlyGovInsertOptions)
+  cover 5 "disable all" (insertOpts == DisableAllInsertOptions)
+  cover 5 "config" isSyncInsertConfig
+  where
+    isSyncInsertConfig =
+      case insertOpts of
+        SyncInsertConfig _ -> True
+        _ -> False

cardano-db-sync/test/Cardano/DbSync/Gen.hs

@@ -118,6 +118,7 @@ syncInsertOptions =
     <$> txOutConfig
     <*> Gen.element [LedgerEnable, LedgerDisable, LedgerIgnore]
     <*> shelleyConfig
+    <*> pure (RewardsConfig True)
     <*> multiAssetConfig
     <*> metadataConfig
     <*> plutusConfig

cardano-db-sync/test/Cardano/DbSyncTest.hs

@@ -18,16 +18,20 @@ tests =
   checkParallel $
     Group
       "Cardano.DbSync"
-      [ ("extractSyncOptions passes tx out", prop_extractSyncOptionsTxOut)
+      [
+        ( "extractSyncOptions passes prune consume migration"
+        , prop_extractSyncOptionsPruneConsumeMigration
+        )
+      , ("extractSyncOptions passes tx out disable", prop_extractSyncOptionsDisable)
       ,
         ( "extractSyncOptions passes offchain pool data"
         , prop_extractSyncOptionsOffchainPoolData
         )
       , ("extractSyncOptions passes governance", prop_extractSyncOptionsGov)
       ]
 
-prop_extractSyncOptionsTxOut :: Property
-prop_extractSyncOptionsTxOut = property $ do
+prop_extractSyncOptionsPruneConsumeMigration :: Property
+prop_extractSyncOptionsPruneConsumeMigration = property $ do
   loggingCfg <- liftIO Logging.empty
   syncNodeParams <- forAll Gen.syncNodeParams
   abortOnPanic <- forAll Gen.bool
@@ -36,17 +40,7 @@ prop_extractSyncOptionsTxOut = property $ do
       (const "SyncNodeConfig") -- SyncNodeConfig does not have Show
       $ Gen.syncNodeConfig loggingCfg
 
-  let isTxOutEnabled' = isTxOutEnabled . sioTxOut . dncInsertOptions $ syncNodeConfig
-      isTxOutDisabled' = isTxOutEnabled . sioTxOut . dncInsertOptions $ syncNodeConfig
-      isTxOutBootstrap' = isTxOutBootstrap . sioTxOut . dncInsertOptions $ syncNodeConfig
-      isTxOutPrune' = isTxOutPrune . sioTxOut . dncInsertOptions $ syncNodeConfig
-      isTxOutConsumed' = isTxOutConsumed . sioTxOut . dncInsertOptions $ syncNodeConfig
-
-  cover 5 "tx out enabled" isTxOutEnabled'
-  cover 5 "tx out disabled" isTxOutDisabled'
-  cover 5 "tx out bootstrap" isTxOutBootstrap'
-  cover 5 "tx out prune" isTxOutPrune'
-  cover 5 "tx out consumed" isTxOutConsumed'
+  coverTxOut syncNodeConfig
 
   let syncOptions = extractSyncOptions syncNodeParams abortOnPanic syncNodeConfig
       expectedPruneConsume =
@@ -74,6 +68,23 @@ prop_extractSyncOptionsOffchainPoolData = property $ do
   ioOffChainPoolData (soptInsertOptions syncOptions)
     === isOffchainPoolDataEnabled (sioOffchainPoolData (dncInsertOptions syncNodeConfig))
 
+prop_extractSyncOptionsDisable :: Property
+prop_extractSyncOptionsDisable = property $ do
+  loggingCfg <- liftIO Logging.empty
+  syncNodeParams <- forAll Gen.syncNodeParams
+  abortOnPanic <- forAll Gen.bool
+  syncNodeConfig <-
+    forAllWith
+      (const "SyncNodeConfig") -- SyncNodeConfig does not have Show
+      $ Gen.syncNodeConfig loggingCfg
+
+  coverTxOut syncNodeConfig
+
+  let isTxOutDisabled' = isTxOutEnabled . sioTxOut . dncInsertOptions $ syncNodeConfig
+      syncOptions = extractSyncOptions syncNodeParams abortOnPanic syncNodeConfig
+
+  ioInOut (soptInsertOptions syncOptions) === isTxOutDisabled'
+
 prop_extractSyncOptionsGov :: Property
 prop_extractSyncOptionsGov = property $ do
   loggingCfg <- liftIO Logging.empty
@@ -88,3 +99,17 @@ prop_extractSyncOptionsGov = property $ do
 
   ioGov (soptInsertOptions syncOptions)
     === isGovernanceEnabled (sioGovernance (dncInsertOptions syncNodeConfig))
+
+coverTxOut :: MonadTest m => SyncNodeConfig -> m ()
+coverTxOut syncNodeConfig = do
+  let isTxOutEnabled' = isTxOutEnabled . sioTxOut . dncInsertOptions $ syncNodeConfig
+      isTxOutDisabled' = isTxOutEnabled . sioTxOut . dncInsertOptions $ syncNodeConfig
+      isTxOutBootstrap' = isTxOutBootstrap . sioTxOut . dncInsertOptions $ syncNodeConfig
+      isTxOutPrune' = isTxOutPrune . sioTxOut . dncInsertOptions $ syncNodeConfig
+      isTxOutConsumed' = isTxOutConsumed . sioTxOut . dncInsertOptions $ syncNodeConfig
+
+  cover 5 "tx out enabled" isTxOutEnabled'
+  cover 5 "tx out disabled" isTxOutDisabled'
+  cover 5 "tx out bootstrap" isTxOutBootstrap'
+  cover 5 "tx out prune" isTxOutPrune'
+  cover 5 "tx out consumed" isTxOutConsumed'