add docs about event log rotation (#2131)

<!-- Describe your change here -->

- Added a new section in docs explaining how event log rotation works,
including usage examples and rough guidance.

- Added `--persistence-rotate-after` as a new optional argument to the
hydra-cluster options, so it can be passed when running with `--busy`
(by default, behavior remains unchanged).

---

<!-- Consider each and tick it off one way or the other -->
* [X] CHANGELOG updated or not needed
* [X] Documentation updated or not needed
* [X] Haddocks updated or not needed
* [X] No new TODOs introduced or explained herafter

---------

Co-authored-by: Noon <[email protected]>

Author: ffakenz

docs/docs/dev/architecture/event-sourcing.md

@@ -36,3 +36,27 @@ When implementing an event source or sink, you might want to consider testing th
   - [ ] Concurrent use of `sourceEvents` is possible
 
 - [ ] General: allocated resources are released (use with/bracket pattern)
+
+### Event Log Rotation
+
+Long-living heads may produce a large number of persisted events, which can impact the restart time of the hydra-node as it needs to read in all the previous to recreate its state.
+
+Event log rotation was introduced to improve recovery times by reducing the number of events that need to be replayed on startup. This is achieved by periodically replacing the current event log with a new one that starts from a checkpoint event, which captures the latest aggregated head state.
+
+Only rotated log files are saved with an incrementing `logId` suffix in their names, while the main `state` log file remains unchanged to preserve backward compatibility. This `logId` suffix corresponds to the ID of the last event included in that file.
+Rotation can be enabled via the optional `--persistence-rotate-after` command-line argument, which specifies the number of events after which rotation should occur.
+> For example, with `--persistence-rotate-after 100`, you’ll get rotated files named: state-99, state-199, state-299, and so on, each containing 100 events. This is because event IDs start at 0.
+
+Note that, depending on the rotation configuration used, the current `state` file may already contain more events than the specified threshold, causing a rotation to occur immediately on startup before any new inputs are processed.
+
+Upon rotation, a server output is produced to notify external agents when a checkpoint occurs, allowing them to perform archival or cleanup actions without interrupting the Hydra Head.
+
+The appropriate value for `--persistence-rotate-after` depends on your specific use case and the expected transaction volume.
+
+> As a rough guideline, in a simple scenario (running a single party on devnet that repeatedly re-spends the same committed UTxO) we observed that setting `--persistence-rotate-after 10000` results in rotated log files of about 8 MB every 3 minutes.
+>
+> Keep in mind that the size and frequency of rotated files will vary depending on several factors:
+>  * Transaction sizes: Larger transactions result in larger event payloads.
+>  * Number of party members: More parties increase the number of L2 protocol messages per snapshot, generating more events.
+>  * Ledger UTxO size: A higher number of UTxOs increases the size of certain events like snapshots.
+>  * Transaction throughput (TPS): Higher TPS leads to more events being produced over time.

hydra-cluster/exe/hydra-cluster/Main.hs

@@ -48,15 +48,15 @@ run options =
             let hydraScriptsTxId = intercalate "," $ toString . serialiseToRawBytesHexText <$> txId
             let envPath = workDir </> ".env"
             writeFile envPath $ "HYDRA_SCRIPTS_TX_ID=" <> hydraScriptsTxId
-            singlePartyOpenAHead tracer workDir backend txId $ \client walletSk _headId -> do
+            singlePartyOpenAHead tracer workDir backend txId persistenceRotateAfter $ \client walletSk _headId -> do
               case scenario of
                 Idle -> forever $ pure ()
                 RespendUTxO -> do
                   -- Start respending the same UTxO with a 100ms delay.
                   -- XXX: Should make this configurable
                   respendUTxO client walletSk 0.1
  where
-  Options{knownNetwork, stateDirectory, publishHydraScripts, useMithril, scenario} = options
+  Options{knownNetwork, stateDirectory, publishHydraScripts, useMithril, scenario, persistenceRotateAfter} = options
 
   withRunningCardanoNode tracer workDir network action =
     findRunningCardanoNode (contramap FromCardanoNode tracer) workDir network >>= \case

hydra-cluster/src/Hydra/Cluster/Options.hs

@@ -6,6 +6,7 @@ import Data.ByteString.Char8 qualified as BSC
 import Data.List qualified as List
 import Hydra.Cardano.Api (TxId, deserialiseFromRawBytesHex)
 import Hydra.Cluster.Fixture (KnownNetwork (..))
+import Hydra.Options (persistenceRotateAfterParser)
 import Hydra.Prelude
 import Options.Applicative (Parser, eitherReader, flag, flag', help, long, metavar, strOption)
 import Options.Applicative.Builder (option)
@@ -16,6 +17,7 @@ data Options = Options
   , publishHydraScripts :: PublishOrReuse
   , useMithril :: UseMithril
   , scenario :: Scenario
+  , persistenceRotateAfter :: Maybe Natural
   }
   deriving stock (Show, Eq, Generic)
   deriving anyclass (ToJSON)
@@ -40,6 +42,7 @@ parseOptions =
     <*> parsePublishHydraScripts
     <*> parseUseMithril
     <*> parseScenario
+    <*> optional persistenceRotateAfterParser
  where
   parseKnownNetwork =
     flag' (Just Preview) (long "preview" <> help "The preview testnet")

hydra-cluster/src/Hydra/Cluster/Scenarios.hs

@@ -106,7 +106,7 @@ import Hydra.Ledger.Cardano (mkSimpleTx, mkTransferTx, unsafeBuildTransaction)
 import Hydra.Ledger.Cardano.Evaluate (maxTxExecutionUnits)
 import Hydra.Logging (Tracer, traceWith)
 import Hydra.Node.DepositPeriod (DepositPeriod (..))
-import Hydra.Options (CardanoChainConfig (..), ChainBackendOptions (..), DirectOptions (..), startChainFrom)
+import Hydra.Options (CardanoChainConfig (..), ChainBackendOptions (..), DirectOptions (..), RunOptions (..), startChainFrom)
 import Hydra.Tx (HeadId, IsTx (balance), Party, txId)
 import Hydra.Tx.ContestationPeriod qualified as CP
 import Hydra.Tx.Utils (dummyValidatorScript, verificationKeyToOnChainId)
@@ -500,10 +500,11 @@ singlePartyOpenAHead ::
   FilePath ->
   backend ->
   [TxId] ->
+  Maybe Natural ->
   -- | Continuation called when the head is open
   (HydraClient -> SigningKey PaymentKey -> HeadId -> IO a) ->
   IO a
-singlePartyOpenAHead tracer workDir backend hydraScriptsTxId callback =
+singlePartyOpenAHead tracer workDir backend hydraScriptsTxId persistenceRotateAfter callback =
   (`finally` returnFundsToFaucet tracer backend Alice) $ do
     refuelIfNeeded tracer backend Alice 25_000_000
     -- Start hydra-node on chain tip
@@ -521,7 +522,9 @@ singlePartyOpenAHead tracer workDir backend hydraScriptsTxId callback =
     utxoToCommit <- seedFromFaucet backend walletVk 100_000_000 (contramap FromFaucet tracer)
 
     let hydraTracer = contramap FromHydraNode tracer
-    withHydraNode hydraTracer aliceChainConfig workDir 1 aliceSk [] [1] $ \n1 -> do
+    options <- prepareHydraNode aliceChainConfig workDir 1 aliceSk [] [] id
+    let options' = options{persistenceRotateAfter}
+    withPreparedHydraNode hydraTracer workDir 1 options' $ \n1 -> do
       -- Initialize & open head
       send n1 $ input "Init" []
       blockTime <- Backend.getBlockTime backend