Merge pull request #6298 from IntersectMBO/bench-master

bench: maintenance

Author: mgmeier

bench/cardano-profile/cardano-profile.cabal

@@ -70,6 +70,7 @@ library
                       , Cardano.Benchmarking.Profile.Primitives
                       , Cardano.Benchmarking.Profile.Vocabulary
                       , Cardano.Benchmarking.Profile.Types
+                      , Cardano.Benchmarking.Profile.Workload.CGroupMemory
                       , Cardano.Benchmarking.Profile.Workload.Latency
                       , Cardano.Benchmarking.Profile.Workload.Voting
   build-depends:        base >=4.12 && <5

bench/cardano-profile/data/all-profiles-coay.json

@@ -23,6 +23,7 @@ import qualified Cardano.Benchmarking.Profile.Playground as Pl (calibrateLoopBlo
 import qualified Cardano.Benchmarking.Profile.Primitives as P
 import qualified Cardano.Benchmarking.Profile.Types as Types
 import qualified Cardano.Benchmarking.Profile.Vocabulary as V
+import qualified Cardano.Benchmarking.Profile.Workload.CGroupMemory as C
 import qualified Cardano.Benchmarking.Profile.Workload.Voting as W
 import qualified Cardano.Benchmarking.Profile.Workload.Latency as L
 
@@ -145,11 +146,18 @@ profilesNoEraCloud =
              -- The name of the defined volume in the Nomad Client config and
              -- where to mount it inside the isolated chroot.
              -- If the volume is not present the deployment will fail!
-             . P.nomadHostVolume (Types.ByNodeType {
+             . P.appendNomadHostVolume (Types.ByNodeType {
                  Types.producer = [Types.HostVolume "/ephemeral" False "ephemeral"]
                , Types.explorer = Nothing
                })
              . P.ssdDirectory "/ephemeral"
+      -- Helper adding workload that takes periodic snapshots of cgroup's `memory.stat`.
+      cgmem = -- Require the cgroup fs mounted by default.
+               P.appendNomadHostVolume (Types.ByNodeType {
+                 Types.producer = [Types.HostVolume "/sys/fs/cgroup" True "cgroup"]
+               , Types.explorer = Nothing
+               })
+             . P.workloadAppend C.cgroupMemoryWorkload
   in [
   -- Value (pre-Voltaire profiles)
     value     & P.name "value-nomadperf"                                   . P.dreps      0 . P.newTracing . P.p2pOn
@@ -163,6 +171,8 @@ profilesNoEraCloud =
   , valueVolt & P.name "value-volt-nomadperf"                              . P.dreps  10000 . P.newTracing . P.p2pOn
   , valueVolt & P.name "value-volt-rtsqg1-nomadperf"                       . P.dreps  10000 . P.newTracing . P.p2pOn . P.rtsGcParallel . P.rtsGcLoadBalance
   , valueVolt & P.name "value-volt-lmdb-nomadperf"                         . P.dreps  10000 . P.newTracing . P.p2pOn . lmdb
+  , valueVolt & P.name "value-volt-cgmem-nomadperf"                        . P.dreps  10000 . P.newTracing . P.p2pOn        . cgmem
+  , valueVolt & P.name "value-volt-lmdb-cgmem-nomadperf"                   . P.dreps  10000 . P.newTracing . P.p2pOn . lmdb . cgmem
   -- Plutus (pre-Voltaire profiles)
   , loop      & P.name "plutus-nomadperf"                                  . P.dreps      0 . P.newTracing . P.p2pOn
   , loop      & P.name "plutus-nomadperf-nop2p"                            . P.dreps      0 . P.newTracing . P.p2pOff
@@ -293,8 +303,8 @@ profilesNoEraCloud =
 
 --------------------------------------------------------------------------------
 
-nomadPerf :: Types.Profile -> Types.Profile
-nomadPerf =
+nomadPerfBase :: Types.Profile -> Types.Profile
+nomadPerfBase =
   -- Exact regions with availability zone (AZ) to match.
   P.regions
     [
@@ -306,20 +316,6 @@ nomadPerf =
   -- Logical cluster separation. To avoid conflicts with same-server machines.
   P.nomadNamespace "perf" . P.nomadClass "perf"
   .
-  -- This will be used as constraints at the Task level.
-  P.nomadResources (Types.ByNodeType {
-    Types.producer = Types.Resources {
-      Types.cores = 8
-    , Types.memory = 15400
-    , Types.memory_max = 16000
-    }
-  , Types.explorer = Just $ Types.Resources {
-      Types.cores = 16
-    , Types.memory = 32000
-    , Types.memory_max = 64000
-    }
-  })
-  .
   -- Instance types will be used as Group "constraints".
   P.awsInstanceTypes (Types.ByNodeType {
     Types.producer = "c5d.2xlarge"
@@ -340,3 +336,28 @@ nomadPerf =
   .
   -- Don't stop the Nomad Job when finished.
   P.clusterKeepRunningOn
+
+nomadPerf :: Types.Profile -> Types.Profile
+nomadPerf =
+  nomadPerfBase . nomadPerfResourcesAll
+
+nomadPerfResourcesAll :: Types.Profile -> Types.Profile
+nomadPerfResourcesAll =
+  -- This will be used as constraints at the Task level.
+  P.nomadResources (Types.ByNodeType {
+    Types.producer = Types.Resources {
+      Types.cores = 8
+    , Types.memory = 15400
+    , Types.memory_max = 16000
+    }
+  -- Explorer is unchanged between cloud profiles.
+  , Types.explorer = Just resourcesExplorer
+  })
+
+resourcesExplorer :: Types.Resources
+resourcesExplorer =
+  Types.Resources {
+    Types.cores = 16
+  , Types.memory = 32000
+  , Types.memory_max = 64000
+  }

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Builtin/Cloud.hs

@@ -100,7 +100,7 @@ clusterNomadSsdNoRegions =
   , Types.explorer = Just $ Types.Resources 16 120000 124000
   })
   .
-  P.nomadHostVolume (
+  P.appendNomadHostVolume (
     let hostVolumes =
           [ Types.HostVolume "/ssd1" False "ssd1"
           , Types.HostVolume "/ssd2" False "ssd2"

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Extra/Scaling.hs

@@ -99,8 +99,8 @@ module Cardano.Benchmarking.Profile.Primitives (
 
   -- Cluster params.
   , clusterMinimunStorage, ssdDirectory, clusterKeepRunningOn
-  , nomadNamespace, nomadClass, nomadResources, nomadHostVolume, nomadSSHLogsOn
-  , awsInstanceTypes, usePublicRouting
+  , nomadNamespace, nomadClass, nomadResources, appendNomadHostVolume
+  , nomadSSHLogsOn, awsInstanceTypes, usePublicRouting
 
   -- Analysis params.
   , analysisOff, analysisStandard, analysisPerformance
@@ -877,8 +877,12 @@ nomadClass nc = nomad (\n -> n {Types.nomad_class = nc})
 nomadResources :: Types.ByNodeType Types.Resources -> Types.Profile -> Types.Profile
 nomadResources r = nomad (\n -> n {Types.resources = r})
 
-nomadHostVolume :: Types.ByNodeType [Types.HostVolume] -> Types.Profile -> Types.Profile
-nomadHostVolume h = nomad (\n -> n {Types.host_volumes = Just h})
+appendNomadHostVolume :: Types.ByNodeType [Types.HostVolume] -> Types.Profile -> Types.Profile
+appendNomadHostVolume h = nomad (\n -> n {Types.host_volumes = Just $
+                                  case Types.host_volumes n of
+                                    Nothing -> h
+                                    (Just bnt) -> bnt <> h
+                                })
 
 nomadSSHLogsOn :: Types.Profile -> Types.Profile
 nomadSSHLogsOn = nomad (\n -> n {Types.fetch_logs_ssh = True})

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Primitives.hs

@@ -670,6 +670,12 @@ data ByNodeType a = ByNodeType
   }
   deriving (Eq, Show, Generic)
 
+instance Semigroup a => Semigroup (ByNodeType a) where
+  (ByNodeType p me) <> (ByNodeType p' me') = ByNodeType (p <> p') (me <> me')
+
+instance Monoid a => Monoid (ByNodeType a) where
+  mempty = ByNodeType mempty Nothing
+
 instance Aeson.ToJSON a => Aeson.ToJSON (ByNodeType a)
 
 instance Aeson.FromJSON a => Aeson.FromJSON (ByNodeType a)

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Types.hs

@@ -0,0 +1,27 @@
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+--------------------------------------------------------------------------------
+
+module Cardano.Benchmarking.Profile.Workload.CGroupMemory (
+  cgroupMemoryWorkload
+) where
+
+--------------------------------------------------------------------------------
+
+import           Prelude
+-- Package: self.
+import qualified Cardano.Benchmarking.Profile.Types as Types
+
+--------------------------------------------------------------------------------
+
+cgroupMemoryWorkload :: Types.Workload
+cgroupMemoryWorkload = Types.Workload {
+    Types.workloadName = "cgroup_memory"
+  , Types.parameters = mempty
+  , Types.entrypoints = Types.Entrypoints {
+      Types.pre_generator = Nothing
+    , Types.producers = "cgroup_memory"
+    }
+  , Types.wait_pools = True
+}

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Workload/CGroupMemory.hs

@@ -1,3 +1,3 @@
 # General reviewers per PR
-#                       Denis            Serge
-*                       @denisshevchenko @deepfire
+#                       Baldur       Michael
+*                       @Icelandjack @mgmeier

cardano-tracer/CODEOWNERS

@@ -746,8 +746,12 @@ let
                     ;
                     # Only for the node that will run the generator
                     withGenerator = taskName == generatorTaskName;
-                    # Only for the tracer task or also nodes if oneTracerPerNode
-                    withTracer = oneTracerPerNode || taskName == "tracer";
+                    # Only if "tracer" was requested.
+                    # And, if tracer task or also nodes if oneTracerPerNode.
+                    withTracer =
+                         profile.node.tracer
+                      && (oneTracerPerNode || taskName == "tracer")
+                    ;
                     inherit withSsh;
                     # ''{{ env "NOMAD_TASK_DIR" }}/supervisor.sock''
                     inherit unixHttpServerPort;

nix/workbench/backend/nomad-job.nix

@@ -166,9 +166,9 @@ let
       };
       tx-generator = rec {
         # Local reference only used if not "cloud".
-        nix-store-path = pkgs.cardanoNodePackages.tx-generator.passthru.noGitRev;
+        nix-store-path = pkgs.cardanoNodePackages.tx-generator;
         flake-reference = "github:intersectmbo/cardano-node";
-        flake-output = "cardanoNodePackages.tx-generator.passthru.noGitRev";
+        flake-output = "cardanoNodePackages.tx-generator";
       };
     }
   ;