Merge pull request #6295 from IntersectMBO/bench-mmap

bench | mmap

Author: mgmeier

README.md

@@ -45,11 +45,11 @@ stateDiagram-v2
 
 The process for getting a `cardano-node` executable can be found in the
 [Cardano Developer
-Portal](https://developers.cardano.org/docs/get-started/cardano-node/installing-cardano-node).
+Portal](https://developers.cardano.org/docs/operate-a-stake-pool/node-operations/installing-cardano-node).
 
 The configuration and files required to run a `cardano-node` in one of the
 supported networks are described also in the [Cardano Developer
-Portal](https://developers.cardano.org/docs/get-started/cardano-node/running-cardano).
+Portal](https://developers.cardano.org/docs/operate-a-stake-pool/node-operations/running-cardano).
 
 # Using `cardano-node` and dependencies as a library
 

cardano-node/cardano-node.cabal

@@ -175,6 +175,7 @@ library
                       , io-classes >= 1.5
                       , iohk-monitoring ^>= 0.2
                       , microlens
+                      , mmap
                       , network-mux
                       , iproute
                       , lobemo-backend-aggregation

cardano-node/src/Cardano/Node/Protocol/Shelley.hs

@@ -48,6 +48,8 @@ import           Control.Monad
 import qualified Data.Aeson as Aeson
 import qualified Data.ByteString as BS
 import qualified Data.Text as T
+import           System.Directory (getFileSize)
+import qualified System.IO.MMap as MMap
 
 ------------------------------------------------------------------------------
 -- Shelley protocol
@@ -94,12 +96,33 @@ readGenesis :: GenesisFile
                        (ShelleyGenesis, GenesisHash)
 readGenesis = readGenesisAny
 
+-- | Read a genesis file using a memory-safe strategy.
+--
+-- Genesis files are used in testing and benchmarking to create testnets with
+-- large datasets. To be able to read these files in memory constrained
+-- environments we use a conditional loading strategy: if file is below ~10
+-- megabytes it is entirely read into memory, otherwise the 'ByteString' is
+-- created using `mmap` that uses the virtual memory subsystem to do on-demand
+-- loading.
+-- With current usage and how benchmakring is done only Shelley and Conway are
+-- affected (Shelley's `initialFunds` and Conway's "delegs" and "initialDReps").
 readGenesisAny :: FromJSON genesis
                => GenesisFile
                -> Maybe GenesisHash
                -> ExceptT GenesisReadError IO (genesis, GenesisHash)
 readGenesisAny (GenesisFile file) mExpectedGenesisHash = do
-    content <- handleIOExceptT (GenesisReadFileError file) $ BS.readFile file
+    content <- handleIOExceptT (GenesisReadFileError file) $ do
+        -- Size of the file in 8-bit bytes.
+        size <- getFileSize file
+        {-- Mainnet files for reference:
+              -byron-genesis.json:   1056360 (1.1M)
+              -shelley-genesis.json:    2486 (2.5K)
+              -alonzo-genesis.json:     9459 (9.3K)
+              -conway-genesis.json:     4168 (4.1K)
+        --}
+        if size >= 10485760 -- 10 megabytes.
+        then MMap.mmapFileByteString file Nothing -- MMap version.
+        else BS.readFile file                     -- Non-lazy version.
     genesisHash <- checkExpectedGenesisHash content mExpectedGenesisHash
     genesis <- firstExceptT (GenesisDecodeError file) $ hoistEither $
                  Aeson.eitherDecodeStrict' content

cardano-node/src/Cardano/Node/Startup.hs

@@ -155,12 +155,13 @@ data BasicInfoCommon = BasicInfoCommon {
   , biNodeStartTime :: UTCTime
   }
 
+-- Fields of this type are made strict to be sure no path from GC roots to genesis is retained
 data BasicInfoShelleyBased = BasicInfoShelleyBased {
-    bisEra               :: Text
-  , bisSystemStartTime   :: UTCTime
-  , bisSlotLength        :: NominalDiffTime
-  , bisEpochLength       :: Word64
-  , bisSlotsPerKESPeriod :: Word64
+    bisEra               :: !Text
+  , bisSystemStartTime   :: !UTCTime
+  , bisSlotLength        :: !NominalDiffTime
+  , bisEpochLength       :: !Word64
+  , bisSlotsPerKESPeriod :: !Word64
   }
 
 data BasicInfoByron = BasicInfoByron {

configuration/cardano/mainnet-config.yaml

@@ -39,7 +39,7 @@ MaxKnownMajorProtocolVersion: 2
 PeerSharing: True
 
 # The following control the number of outbound connections to strictly our upstream peers
-# cf. https://developers.cardano.org/docs/get-started/cardano-node/p2p
+# cf. https://developers.cardano.org/docs/operate-a-stake-pool/node-operations/topology
 # defaults are provided by ouroboros-network, but can be overridden by uncommenting below:
 # SyncTargetNumberOfActiveBigLedgerPeers: 30
 # SyncTargetNumberOfActivePeers: 5