wb | add optional cgroup memory workload

Author: fmaste

bench/cardano-profile/cardano-profile.cabal

@@ -70,6 +70,7 @@ library
                       , Cardano.Benchmarking.Profile.Primitives
                       , Cardano.Benchmarking.Profile.Vocabulary
                       , Cardano.Benchmarking.Profile.Types
+                      , Cardano.Benchmarking.Profile.Workload.CGroupMemory
                       , Cardano.Benchmarking.Profile.Workload.Latency
                       , Cardano.Benchmarking.Profile.Workload.Voting
   build-depends:        base >=4.12 && <5

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Builtin/Cloud.hs

@@ -145,7 +145,7 @@ profilesNoEraCloud =
              -- The name of the defined volume in the Nomad Client config and
              -- where to mount it inside the isolated chroot.
              -- If the volume is not present the deployment will fail!
-             . P.nomadHostVolume (Types.ByNodeType {
+             . P.appendNomadHostVolume (Types.ByNodeType {
                  Types.producer = [Types.HostVolume "/ephemeral" False "ephemeral"]
                , Types.explorer = Nothing
                })
@@ -293,8 +293,8 @@ profilesNoEraCloud =
 
 --------------------------------------------------------------------------------
 
-nomadPerf :: Types.Profile -> Types.Profile
-nomadPerf =
+nomadPerfBase :: Types.Profile -> Types.Profile
+nomadPerfBase =
   -- Exact regions with availability zone (AZ) to match.
   P.regions
     [
@@ -306,20 +306,6 @@ nomadPerf =
   -- Logical cluster separation. To avoid conflicts with same-server machines.
   P.nomadNamespace "perf" . P.nomadClass "perf"
   .
-  -- This will be used as constraints at the Task level.
-  P.nomadResources (Types.ByNodeType {
-    Types.producer = Types.Resources {
-      Types.cores = 8
-    , Types.memory = 15400
-    , Types.memory_max = 16000
-    }
-  , Types.explorer = Just $ Types.Resources {
-      Types.cores = 16
-    , Types.memory = 32000
-    , Types.memory_max = 64000
-    }
-  })
-  .
   -- Instance types will be used as Group "constraints".
   P.awsInstanceTypes (Types.ByNodeType {
     Types.producer = "c5d.2xlarge"
@@ -340,3 +326,28 @@ nomadPerf =
   .
   -- Don't stop the Nomad Job when finished.
   P.clusterKeepRunningOn
+
+nomadPerf :: Types.Profile -> Types.Profile
+nomadPerf =
+  nomadPerfBase . nomadPerfResourcesAll
+
+nomadPerfResourcesAll :: Types.Profile -> Types.Profile
+nomadPerfResourcesAll =
+  -- This will be used as constraints at the Task level.
+  P.nomadResources (Types.ByNodeType {
+    Types.producer = Types.Resources {
+      Types.cores = 8
+    , Types.memory = 15400
+    , Types.memory_max = 16000
+    }
+  -- Explorer is unchanged between cloud profiles.
+  , Types.explorer = Just resourcesExplorer
+  })
+
+resourcesExplorer :: Types.Resources
+resourcesExplorer =
+  Types.Resources {
+    Types.cores = 16
+  , Types.memory = 32000
+  , Types.memory_max = 64000
+  }

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Extra/Scaling.hs

@@ -100,7 +100,7 @@ clusterNomadSsdNoRegions =
   , Types.explorer = Just $ Types.Resources 16 120000 124000
   })
   .
-  P.nomadHostVolume (
+  P.appendNomadHostVolume (
     let hostVolumes =
           [ Types.HostVolume "/ssd1" False "ssd1"
           , Types.HostVolume "/ssd2" False "ssd2"

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Primitives.hs

@@ -99,8 +99,8 @@ module Cardano.Benchmarking.Profile.Primitives (
 
   -- Cluster params.
   , clusterMinimunStorage, ssdDirectory, clusterKeepRunningOn
-  , nomadNamespace, nomadClass, nomadResources, nomadHostVolume, nomadSSHLogsOn
-  , awsInstanceTypes, usePublicRouting
+  , nomadNamespace, nomadClass, nomadResources, appendNomadHostVolume
+  , nomadSSHLogsOn, awsInstanceTypes, usePublicRouting
 
   -- Analysis params.
   , analysisOff, analysisStandard, analysisPerformance
@@ -877,8 +877,12 @@ nomadClass nc = nomad (\n -> n {Types.nomad_class = nc})
 nomadResources :: Types.ByNodeType Types.Resources -> Types.Profile -> Types.Profile
 nomadResources r = nomad (\n -> n {Types.resources = r})
 
-nomadHostVolume :: Types.ByNodeType [Types.HostVolume] -> Types.Profile -> Types.Profile
-nomadHostVolume h = nomad (\n -> n {Types.host_volumes = Just h})
+appendNomadHostVolume :: Types.ByNodeType [Types.HostVolume] -> Types.Profile -> Types.Profile
+appendNomadHostVolume h = nomad (\n -> n {Types.host_volumes = Just $
+                                  case Types.host_volumes n of
+                                    Nothing -> h
+                                    (Just bnt) -> bnt <> h
+                                })
 
 nomadSSHLogsOn :: Types.Profile -> Types.Profile
 nomadSSHLogsOn = nomad (\n -> n {Types.fetch_logs_ssh = True})

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Types.hs

@@ -670,6 +670,12 @@ data ByNodeType a = ByNodeType
   }
   deriving (Eq, Show, Generic)
 
+instance Semigroup a => Semigroup (ByNodeType a) where
+  (ByNodeType p me) <> (ByNodeType p' me') = ByNodeType (p <> p') (me <> me')
+
+instance Monoid a => Monoid (ByNodeType a) where
+  mempty = ByNodeType mempty Nothing
+
 instance Aeson.ToJSON a => Aeson.ToJSON (ByNodeType a)
 
 instance Aeson.FromJSON a => Aeson.FromJSON (ByNodeType a)

bench/cardano-profile/src/Cardano/Benchmarking/Profile/Workload/CGroupMemory.hs

@@ -0,0 +1,27 @@
+{-# LANGUAGE Trustworthy #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+--------------------------------------------------------------------------------
+
+module Cardano.Benchmarking.Profile.Workload.CGroupMemory (
+  cgroupMemoryWorkload
+) where
+
+--------------------------------------------------------------------------------
+
+import           Prelude
+-- Package: self.
+import qualified Cardano.Benchmarking.Profile.Types as Types
+
+--------------------------------------------------------------------------------
+
+cgroupMemoryWorkload :: Types.Workload
+cgroupMemoryWorkload = Types.Workload {
+    Types.workloadName = "cgroup_memory"
+  , Types.parameters = mempty
+  , Types.entrypoints = Types.Entrypoints {
+      Types.pre_generator = Nothing
+    , Types.producers = "cgroup_memory"
+    }
+  , Types.wait_pools = True
+}

nix/workbench/backend/nomad.sh

@@ -3904,6 +3904,12 @@ client {
     disable_file_sandbox = false
   }
 
+  # Feature parity with the "nomadperf" cloud cluster.
+  host_volume "cgroup" {
+    path = "/sys/fs/cgroup"
+    read_only = true
+  }
+
   # To use when resolving IPs.
   #meta {
   #  my_ip = "${addr}"

nix/workbench/workload/cgroup_memory.nix

@@ -0,0 +1,107 @@
+{ pkgs
+, profile
+, nodeSpecs
+, workload
+}:
+
+with pkgs.lib;
+
+let
+  bashInteractive = pkgs.bashInteractive;
+  coreutils       = pkgs.coreutils;
+  jq              = pkgs.jq;
+# Assumptions:
+# - Command `date` and node's log use the same timezone!
+in ''
+#!${bashInteractive}/bin/sh
+
+######################################################################
+# Set script globals #################################################
+######################################################################
+
+# Strict runtime
+################
+
+# e:        Immediately exit if any command has a non-zero exit status
+# u:        Reference to non previously defined variables is an error
+# pipefail: Any failed command in a pipeline is used as return code
+set -euo pipefail
+
+# Fetch all defined node names (Including "explorer" nodes)
+###########################################################
+
+node_specs_nodes=$(${jq}/bin/jq --raw-output \
+  "keys | join (\" \")"                      \
+  ../../node-specs.json                      \
+)
+node_specs_pools=$(${jq}/bin/jq              \
+  'map(select(.kind == "pool")) | length'    \
+  ../../node-specs.json                      \
+)
+${coreutils}/bin/echo "node-specs.json:"
+${coreutils}/bin/echo "- Nodes: [''${node_specs_nodes[*]}]"
+${coreutils}/bin/echo "- Pools: ''${node_specs_pools}"
+
+# Look for locally deployed nodes and save starting time
+########################################################
+
+nodes=()
+started_time=$(${coreutils}/bin/date +%s)
+for node in ''${node_specs_nodes[*]}
+do
+  if test -d "../../''${node}"
+  then
+    nodes+=("''${node}")
+    # Save the starting time
+    ${coreutils}/bin/echo "''${started_time}" > "./start_time_''${node}"
+  fi
+done
+${coreutils}/bin/echo "Found deployed nodes:"
+${coreutils}/bin/echo "- Nodes: [''${nodes[*]}]"
+
+######################################################################
+# Main ###############################################################
+######################################################################
+
+# The entrypoint function.
+function cgroup_memory() {
+
+  msg "Started!"
+
+  local jobs_array=()
+  for node in ''${nodes[*]}
+  do
+    cgroup_memory_deployed "''${node}" &
+    jobs_array+=("$!")
+  done
+  wait "''${jobs_array[@]}"
+}
+
+function cgroup_memory_deployed() {
+  local node=$1
+
+  msg "Starting node: ''${node}"
+
+  local cgroup
+  cgroup="$(${coreutils}/bin/cat /proc/self/cgroup | ${coreutils}/bin/cut -d: -f3)"
+  msg "CGroup found for \"''${node}\": ''${cgroup}"
+
+  while true
+  do
+      ${coreutils}/bin/cat /sys/fs/cgroup"''${cgroup}"/memory.stat \
+    > "./memory.stat_''${node}-''$(${coreutils}/bin/date +"%Y-%m-%d-%H-%M-%S-%3N")"
+    ${coreutils}/bin/sleep 60
+  done
+}
+
+######################################################################
+# Utils ##############################################################
+######################################################################
+
+function msg {
+  # Outputs to stdout, unbuffered if not the message may be lost!
+  ${coreutils}/bin/stdbuf -o0 \
+    ${bashInteractive}/bin/sh -c \
+      "${coreutils}/bin/echo -e \"$(${coreutils}/bin/date --rfc-3339=seconds): $1\""
+}
+''