Updated config files of benchmark_suite applications

Author: giltirn

benchmark_suite/cupti_gpu_kernel_outlier/chimbuko_config.sh

@@ -8,7 +8,7 @@ service_node_iface=eth0 #network interface upon which communication to the servi
 ####################################
 #Options for visualization module
 ####################################
-use_viz=1 #enable or disable the visualization
+use_viz=0 #enable or disable the visualization
 viz_root=/opt/chimbuko/viz    #the root directory of the visualization module  <------------ ***SET ME (if using viz)***
 viz_worker_port=6379  #the port on which to run the redis server for the visualization backend
 viz_port=5002 #the port on which to run the webserver
@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/cupti_gpu_kernel_outlier_multistream/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/mpi_comm_outlier/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -57,7 +60,7 @@ ad_outlier_sstd_sigma=12 #number of standard deviations that defines an outlier
 #      command to launch the AD (output to chimbuko/vars/chimbuko_ad_cmdline.var); they can be overridden by the run script if desired providing the appropriate modifications
 #      are made to the AD launch command. The remainder of the variables are used only by TAU and can be freely overridden.
 ####################################
-export TAU_ADIOS2_ENGINE=SST    #online communication engine (alternative BP4 although this goes through the disk system and may be slower unless the BPfiles are stored on a burst disk)
+export TAU_ADIOS2_ENGINE=BP4    #online communication engine (alternative BP4 although this goes through the disk system and may be slower unless the BPfiles are stored on a burst disk)
 export TAU_ADIOS2_ONE_FILE=FALSE  #a different connection file for each rank
 export TAU_ADIOS2_PERIODIC=1   #enable/disable ADIOS2 periodic output
 export TAU_ADIOS2_PERIOD=1000000  #period in us between ADIOS2 io steps
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/mpi_threaded_comm_outlier/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/multiinstance_nompi/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/openmp_cpu/chimbuko_config.sh

@@ -8,7 +8,7 @@ service_node_iface=eth0 #network interface upon which communication to the servi
 ####################################
 #Options for visualization module
 ####################################
-use_viz=0 #enable or disable the visualization
+use_viz=1 #enable or disable the visualization
 viz_root=/opt/chimbuko/viz    #the root directory of the visualization module  <------------ ***SET ME (if using viz)***
 viz_worker_port=6379  #the port on which to run the redis server for the visualization backend
 viz_port=5002 #the port on which to run the webserver
@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/python_hello/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

benchmark_suite/simple_workflow/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi