Skip to content

Commit 97bbf3f

Browse files
CmdvCmdv
committed
improve pipelines, add stop_at_block config
1 parent 7663150 commit 97bbf3f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+1104
-2828
lines changed

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ getDBSyncPGPass = enpPGPassSource . dbSyncParams
231231

232232
queryDBSync :: DBSyncEnv -> DB.DbM a -> IO a
233233
queryDBSync env = do
234-
DB.runDbMTransactionNoLogging (getDBSyncPGPass env)
234+
DB.runDbStandaloneTransSilent (getDBSyncPGPass env)
235235

236236
getPoolLayer :: DBSyncEnv -> IO PoolDataLayer
237237
getPoolLayer env = do
@@ -384,6 +384,8 @@ emptyMetricsSetters =
384384
, metricsSetDbQueueLength = \_ -> pure ()
385385
, metricsSetDbBlockHeight = \_ -> pure ()
386386
, metricsSetDbSlotHeight = \_ -> pure ()
387+
, metricsSetDbEpochSyncDuration = \_ -> pure ()
388+
, metricsSetDbEpochSyncNumber = \_ -> pure ()
387389
}
388390

389391
withFullConfig ::

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ insertConfig = do
3434
, sioPoolStats = PoolStatsConfig False
3535
, sioJsonType = JsonTypeDisable
3636
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
37+
, sioStopAtBlock = Nothing
3738
}
3839

3940
dncInsertOptions cfg @?= expected

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ insertConfig = do
104104
, sioPoolStats = PoolStatsConfig False
105105
, sioJsonType = JsonTypeDisable
106106
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
107+
, sioStopAtBlock = Nothing
107108
}
108109

109110
dncInsertOptions cfg @?= expected

‎cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Schema.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ validateSchemaColumns =
7979
validateCall dbSync (Proxy @DB.MultiAsset)
8080
validateCall dbSync (Proxy @DB.MaTxMint)
8181

82-
-- Cardano.Db.Schema.Core.StakeDeligation
82+
-- Cardano.Db.Schema.Core.StakeDelegation
8383
validateCall dbSync (Proxy @DB.StakeAddress)
8484
validateCall dbSync (Proxy @DB.StakeRegistration)
8585
validateCall dbSync (Proxy @DB.StakeDeregistration)

‎cardano-db-sync/app/test-http-get-json-metadata.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import Network.HTTP.Client.TLS (tlsManagerSettings)
3131
main :: IO ()
3232
main = do
3333
manager <- Http.newManager tlsManagerSettings
34-
xs <- DB.runDbMTransactionNoLogging DB.PGPassDefaultEnv queryTestOffChainData
34+
xs <- DB.runDbStandaloneTransSilent DB.PGPassDefaultEnv queryTestOffChainData
3535
putStrLn $ "testOffChainPoolDataFetch: " ++ show (length xs) ++ " tests to run."
3636
tfs <- foldM (testOne manager) emptyTestFailure xs
3737
reportTestFailures tfs

‎cardano-db-sync/src/Cardano/DbSync.hs

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ import Cardano.DbSync.Era
5555
import Cardano.DbSync.Error
5656
import Cardano.DbSync.Ledger.State
5757
import Cardano.DbSync.OffChain (runFetchOffChainPoolThread, runFetchOffChainVoteThread)
58-
import Cardano.DbSync.Rollback (unsafeRollback)
58+
import Cardano.DbSync.Rollback (handlePostRollbackSnapshots, unsafeRollback)
5959
import Cardano.DbSync.Sync (runSyncNodeClient)
6060
import Cardano.DbSync.Tracing.ToObjectOrphans ()
6161
import Cardano.DbSync.Types
@@ -98,7 +98,7 @@ runMigrationsOnly knownMigrations trce params syncNodeConfigFromFile = do
9898
msg <- DB.getMaintenancePsqlConf pgConfig
9999
logInfo trce $ "Running database migrations in mode " <> textShow mode
100100
logInfo trce msg
101-
DB.runMigrations pgConfig True dbMigrationDir (Just $ DB.LogFileDir "/tmp") mode (txOutConfigToTableType txOutConfig)
101+
DB.runMigrations (Just trce) pgConfig True dbMigrationDir (Just $ DB.LogFileDir "/tmp") mode (txOutConfigToTableType txOutConfig)
102102

103103
-- Always run Initial mode only - never indexes
104104
(ranMigrations, unofficial) <- runMigration DB.Initial
@@ -150,7 +150,7 @@ runDbSync metricsSetters iomgr trce params syncNodeConfigFromFile abortOnPanic =
150150
logInfo trce $ "Running NearTip database migrations in mode " <> textShow mode
151151
logInfo trce msg
152152
when (mode `elem` [DB.NearTip, DB.Full]) $ logWarning trce indexesMsg
153-
DB.runMigrations pgConfig True dbMigrationDir (Just $ DB.LogFileDir "/tmp") mode (txOutConfigToTableType txOutConfig)
153+
DB.runMigrations (Just trce) pgConfig True dbMigrationDir (Just $ DB.LogFileDir "/tmp") mode (txOutConfigToTableType txOutConfig)
154154

155155
runSyncNode
156156
metricsSetters
@@ -199,7 +199,7 @@ runSyncNode metricsSetters trce iomgr dbConnSetting runNearTipMigrationFnc syncN
199199
let useLedger = shouldUseLedger (sioLedger $ dncInsertOptions syncNodeConfigFromFile)
200200
-- The main thread
201201
bracket
202-
(acquireDbConnection [dbConnSetting])
202+
(DB.acquireConnection [dbConnSetting])
203203
HsqlC.release
204204
( \dbConn -> do
205205
runOrThrowIO $ runExceptT $ do
@@ -216,6 +216,7 @@ runSyncNode metricsSetters trce iomgr dbConnSetting runNearTipMigrationFnc syncN
216216
syncEnv <-
217217
ExceptT $
218218
mkSyncEnvFromConfig
219+
metricsSetters
219220
trce
220221
dbEnv
221222
syncOptions
@@ -239,12 +240,15 @@ runSyncNode metricsSetters trce iomgr dbConnSetting runNearTipMigrationFnc syncN
239240
DB.noLedgerMigrations dbEnv trce
240241
insertValidateGenesisDist syncEnv (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile)
241242

243+
-- Handle ledger snapshots after rollback to ensure consistency
244+
liftIO $ handlePostRollbackSnapshots syncEnv (enpMaybeRollback syncNodeParams)
245+
242246
-- communication channel between datalayer thread and chainsync-client thread
243247
threadChannels <- liftIO newThreadChannels
244248
liftIO $
245249
race_
246250
-- We split the main thread into two parts to allow for graceful shutdown of the main App db thread.
247-
(runDbThread syncEnv metricsSetters threadChannels)
251+
(runDbThread syncEnv threadChannels)
248252
( mapConcurrently_
249253
id
250254
[ runSyncNodeClient metricsSetters syncEnv iomgr trce threadChannels (enpSocketPath syncNodeParams)

‎cardano-db-sync/src/Cardano/DbSync/Api.hs

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ runConsumedTxOutMigrationsMaybe syncEnv = do
143143
let pcm = getPruneConsume syncEnv
144144
txOutVariantType = getTxOutVariantType syncEnv
145145
logInfo (getTrace syncEnv) $ "runConsumedTxOutMigrationsMaybe: " <> textShow pcm
146-
DB.runDbTransactionIohkNoLogging (envDbEnv syncEnv) $
146+
DB.runDbDirectSilent (envDbEnv syncEnv) $
147147
DB.runConsumedTxOutMigrations
148148
(getTrace syncEnv)
149149
maxBulkSize
@@ -153,11 +153,11 @@ runConsumedTxOutMigrationsMaybe syncEnv = do
153153

154154
runAddJsonbToSchema :: SyncEnv -> IO ()
155155
runAddJsonbToSchema syncEnv =
156-
void $ DB.runDbTransactionIohkNoLogging (envDbEnv syncEnv) DB.enableJsonbInSchema
156+
void $ DB.runDbDirectSilent (envDbEnv syncEnv) DB.enableJsonbInSchema
157157

158158
runRemoveJsonbFromSchema :: SyncEnv -> IO ()
159159
runRemoveJsonbFromSchema syncEnv =
160-
void $ DB.runDbTransactionIohkNoLogging (envDbEnv syncEnv) DB.disableJsonbInSchema
160+
void $ DB.runDbDirectSilent (envDbEnv syncEnv) DB.disableJsonbInSchema
161161

162162
getSafeBlockNoDiff :: SyncEnv -> Word64
163163
getSafeBlockNoDiff syncEnv = 2 * getSecurityParam syncEnv
@@ -243,7 +243,7 @@ getInsertOptions :: SyncEnv -> InsertOptions
243243
getInsertOptions = soptInsertOptions . envOptions
244244

245245
getSlotHash :: DB.DbEnv -> SlotNo -> IO [(SlotNo, ByteString)]
246-
getSlotHash backend = DB.runDbTransactionIohkNoLogging backend . DB.querySlotHash
246+
getSlotHash backend = DB.runDbDirectSilent backend . DB.querySlotHash
247247

248248
hasLedgerState :: SyncEnv -> Bool
249249
hasLedgerState syncEnv =
@@ -254,7 +254,7 @@ hasLedgerState syncEnv =
254254
getDbLatestBlockInfo :: DB.DbEnv -> IO (Maybe TipInfo)
255255
getDbLatestBlockInfo dbEnv = do
256256
runMaybeT $ do
257-
block <- MaybeT $ DB.runDbTransactionIohkNoLogging dbEnv DB.queryLatestBlock
257+
block <- MaybeT $ DB.runDbDirectSilent dbEnv DB.queryLatestBlock
258258
-- The EpochNo, SlotNo and BlockNo can only be zero for the Byron
259259
-- era, but we need to make the types match, hence `fromMaybe`.
260260
pure $
@@ -297,6 +297,7 @@ getCurrentTipBlockNo env = do
297297
Nothing -> pure Origin
298298

299299
mkSyncEnv ::
300+
MetricSetters ->
300301
Trace IO Text ->
301302
DB.DbEnv ->
302303
SyncOptions ->
@@ -308,18 +309,18 @@ mkSyncEnv ::
308309
SyncNodeParams ->
309310
RunMigration ->
310311
IO SyncEnv
311-
mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfigFromFile syncNP runNearTipMigrationFnc = do
312-
dbCNamesVar <- newTVarIO =<< DB.runDbTransactionIohkNoLogging dbEnv DB.queryRewardAndEpochStakeConstraints
312+
mkSyncEnv metricSetters trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfigFromFile syncNP runNearTipMigrationFnc = do
313+
dbCNamesVar <- newTVarIO =<< DB.runDbDirectSilent dbEnv DB.queryRewardAndEpochStakeConstraints
313314
cache <-
314315
if soptCache syncOptions
315316
then
316317
newEmptyCache
317318
CacheCapacity
318-
{ cacheCapacityAddress = 100000
319-
, cacheCapacityStake = 100000
320-
, cacheCapacityDatum = 250000
321-
, cacheCapacityMultiAsset = 250000
322-
, cacheCapacityTx = 100000
319+
{ cacheCapacityAddress = 50000
320+
, cacheCapacityStake = 50000
321+
, cacheCapacityDatum = 125000
322+
, cacheCapacityMultiAsset = 125000
323+
, cacheCapacityTx = 50000
323324
}
324325
else pure useNoCache
325326
consistentLevelVar <- newTVarIO Unchecked
@@ -356,6 +357,7 @@ mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfig
356357
pure $
357358
SyncEnv
358359
{ envDbEnv = dbEnv
360+
, envMetricSetters = metricSetters
359361
, envBootstrap = bootstrapVar
360362
, envCache = cache
361363
, envEpochStatistics = epochStatistics
@@ -379,6 +381,7 @@ mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfig
379381
isTxOutConsumedBootstrap' = isTxOutConsumedBootstrap . sioTxOut . dncInsertOptions
380382

381383
mkSyncEnvFromConfig ::
384+
MetricSetters ->
382385
Trace IO Text ->
383386
DB.DbEnv ->
384387
SyncOptions ->
@@ -388,7 +391,7 @@ mkSyncEnvFromConfig ::
388391
-- | run migration function
389392
RunMigration ->
390393
IO (Either SyncNodeError SyncEnv)
391-
mkSyncEnvFromConfig trce dbEnv syncOptions genCfg syncNodeConfigFromFile syncNodeParams runNearTipMigrationFnc =
394+
mkSyncEnvFromConfig metricsSetters trce dbEnv syncOptions genCfg syncNodeConfigFromFile syncNodeParams runNearTipMigrationFnc =
392395
case genCfg of
393396
GenesisCardano _ bCfg sCfg _ _
394397
| unProtocolMagicId (Byron.configProtocolMagicId bCfg) /= Shelley.sgNetworkMagic (scConfig sCfg) ->
@@ -414,6 +417,7 @@ mkSyncEnvFromConfig trce dbEnv syncOptions genCfg syncNodeConfigFromFile syncNod
414417
| otherwise ->
415418
Right
416419
<$> mkSyncEnv
420+
metricsSetters
417421
trce
418422
dbEnv
419423
syncOptions
@@ -434,7 +438,7 @@ getLatestPoints env = do
434438
verifySnapshotPoint env snapshotPoints
435439
NoLedger _ -> do
436440
-- Brings the 5 latest.
437-
lastPoints <- DB.runDbTransactionIohkNoLogging (envDbEnv env) DB.queryLatestPoints
441+
lastPoints <- DB.runDbDirectSilent (envDbEnv env) DB.queryLatestPoints
438442
pure $ mapMaybe convert lastPoints
439443
where
440444
convert (Nothing, _) = Nothing
@@ -489,7 +493,7 @@ getBootstrapInProgress ::
489493
DB.DbEnv ->
490494
IO Bool
491495
getBootstrapInProgress trce bootstrapFlag dbEnv = do
492-
DB.runDbTransactionIohkNoLogging dbEnv $ do
496+
DB.runDbDirectSilent dbEnv $ do
493497
ems <- DB.queryAllExtraMigrations
494498
let btsState = DB.bootstrapState ems
495499
case (bootstrapFlag, btsState) of

‎cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ storePage syncEnv percQuantum (n, ls) = do
130130
txOuts <- mapM (prepareTxOut syncEnv) ls
131131
txOutIds <- lift $ DB.insertBulkTxOut False $ etoTxOut . fst <$> txOuts
132132
let maTxOuts = concatMap (mkmaTxOuts txOutVariantType) $ zip txOutIds (snd <$> txOuts)
133-
void . lift $ DB.insertBulkMaTxOut maTxOuts
133+
void . lift $ DB.insertBulkMaTxOutPiped [maTxOuts]
134134
where
135135
txOutVariantType = getTxOutVariantType syncEnv
136136
trce = getTrace syncEnv

‎cardano-db-sync/src/Cardano/DbSync/Api/Types.hs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import Cardano.DbSync.Config.Types (SyncNodeConfig)
3232
import Cardano.DbSync.Ledger.Types (HasLedgerEnv)
3333
import Cardano.DbSync.LocalStateQuery (NoLedgerEnv)
3434
import Cardano.DbSync.Types (
35+
MetricSetters,
3536
OffChainPoolResult,
3637
OffChainPoolWorkQueue,
3738
OffChainVoteResult,
@@ -41,6 +42,7 @@ import Cardano.DbSync.Types (
4142
-- | SyncEnv is the main environment for the whole application.
4243
data SyncEnv = SyncEnv
4344
{ envDbEnv :: !DB.DbEnv
45+
, envMetricSetters :: !MetricSetters
4446
, envCache :: !CacheStatus
4547
, envEpochStatistics :: !(StrictTVar IO EpochStatistics)
4648
, envConsistentLevel :: !(StrictTVar IO ConsistentLevel)

‎cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ data SyncInsertOptions = SyncInsertOptions
191191
, sioPoolStats :: PoolStatsConfig
192192
, sioJsonType :: JsonTypeConfig
193193
, sioRemoveJsonbFromSchema :: RemoveJsonbFromSchemaConfig
194+
, sioStopAtBlock :: Maybe Word64
194195
}
195196
deriving (Eq, Show)
196197

@@ -458,6 +459,7 @@ parseOverrides obj baseOptions = do
458459
<*> obj .:? "pool_stat" .!= sioPoolStats baseOptions
459460
<*> obj .:? "json_type" .!= sioJsonType baseOptions
460461
<*> obj .:? "remove_jsonb_from_schema" .!= sioRemoveJsonbFromSchema baseOptions
462+
<*> obj .:? "stop_at_block" .!= sioStopAtBlock baseOptions
461463

462464
instance ToJSON SyncInsertConfig where
463465
toJSON (SyncInsertConfig preset options) =
@@ -479,6 +481,7 @@ optionsToList SyncInsertOptions {..} =
479481
, toJsonIfSet "pool_stat" sioPoolStats
480482
, toJsonIfSet "json_type" sioJsonType
481483
, toJsonIfSet "remove_jsonb_from_schema" sioRemoveJsonbFromSchema
484+
, toJsonIfSet "stop_at_block" sioStopAtBlock
482485
]
483486

484487
toJsonIfSet :: ToJSON a => Text -> a -> Maybe Pair
@@ -500,6 +503,7 @@ instance FromJSON SyncInsertOptions where
500503
<*> obj .:? "pool_stat" .!= sioPoolStats def
501504
<*> obj .:? "json_type" .!= sioJsonType def
502505
<*> obj .:? "remove_jsonb_from_schema" .!= sioRemoveJsonbFromSchema def
506+
<*> obj .:? "stop_at_block" .!= sioStopAtBlock def
503507

504508
instance ToJSON SyncInsertOptions where
505509
toJSON SyncInsertOptions {..} =
@@ -516,6 +520,7 @@ instance ToJSON SyncInsertOptions where
516520
, "pool_stat" .= sioPoolStats
517521
, "json_type" .= sioJsonType
518522
, "remove_jsonb_from_schema" .= sioRemoveJsonbFromSchema
523+
, "stop_at_block" .= sioStopAtBlock
519524
]
520525

521526
instance ToJSON RewardsConfig where
@@ -745,6 +750,7 @@ instance Default SyncInsertOptions where
745750
, sioPoolStats = PoolStatsConfig False
746751
, sioJsonType = JsonTypeText
747752
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
753+
, sioStopAtBlock = Nothing
748754
}
749755

750756
fullInsertOptions :: SyncInsertOptions
@@ -763,6 +769,7 @@ fullInsertOptions =
763769
, sioPoolStats = PoolStatsConfig True
764770
, sioJsonType = JsonTypeText
765771
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
772+
, sioStopAtBlock = Nothing
766773
}
767774

768775
onlyUTxOInsertOptions :: SyncInsertOptions
@@ -781,6 +788,7 @@ onlyUTxOInsertOptions =
781788
, sioPoolStats = PoolStatsConfig False
782789
, sioJsonType = JsonTypeText
783790
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
791+
, sioStopAtBlock = Nothing
784792
}
785793

786794
onlyGovInsertOptions :: SyncInsertOptions
@@ -807,6 +815,7 @@ disableAllInsertOptions =
807815
, sioGovernance = GovernanceConfig False
808816
, sioJsonType = JsonTypeText
809817
, sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
818+
, sioStopAtBlock = Nothing
810819
}
811820

812821
addressTypeToEnableDisable :: IsString s => TxOutVariantType -> s

0 commit comments

Comments
 (0)