diff --git a/darshan-runtime/configure.ac b/darshan-runtime/configure.ac index b36444c8a..4d3618410 100644 --- a/darshan-runtime/configure.ac +++ b/darshan-runtime/configure.ac @@ -598,14 +598,6 @@ if test "x$enable_darshan_runtime" = xyes ; then AC_MSG_ERROR(must provide --with-jobid-env= argument to configure.) fi - __DARSHAN_LOG_HINTS_DEFAULT="romio_no_indep_rw=true;cb_nodes=4" - if test "x$GOT_LOG_HINTS" != xyes ; then - dnl use default hints - AC_DEFINE_UNQUOTED([__DARSHAN_LOG_HINTS], "$__DARSHAN_LOG_HINTS_DEFAULT", - [Comma-separated list of MPI-IO hints for log file write]) - __DARSHAN_LOG_HINTS=$__DARSHAN_LOG_HINTS_DEFAULT - fi - # checks to see how we can print 64 bit values on this architecture gt_INTTYPES_PRI if test "x$PRI_MACROS_BROKEN" = x1; then @@ -694,6 +686,48 @@ if test "x$enable_darshan_runtime" = xyes ; then if test "x$is_ompi" = x1 ; then AC_DEFINE(HAVE_OPEN_MPI, 1, [Define if OpenMPI is being used]) + + dnl Check if version is 5.0.6 or later + AC_MSG_CHECKING([Check if OpenMPI version is 5.0.6 or later]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +#if (OMPI_MAJOR_VERSION*1000000 + OMPI_MINOR_VERSION*1000 + OMPI_RELEASE_VERSION < 5000006) + choke me +#endif + ]])], [ompi_ge_5_0_6=1], [ompi_ge_5_0_6=0]) + AC_MSG_RESULT([$ompi_ge_5_0_6]) + AC_DEFINE_UNQUOTED(OMPI_GE_5_0_6, [$ompi_ge_5_0_6], + [Define if OpenMPI version is 5.0.6 or later]) + fi + + if test "x$GOT_LOG_HINTS" != xyes ; then + # set default hints + __DARSHAN_LOG_HINTS_DEFAULT="romio_no_indep_rw=true;cb_nodes=4" + + if test "x$is_ompi" = x1 ; then + # OpenMPI 5.0.5 and priors contain a bug that can corrupt the + # Darshan log files. The bug is related to file locking protocols + # implemented when data sieving is enabled. Setting hint cb_nodes + # to 1 can avoid the bug, but may make writing log files slow. + # For more information, see PR 1070, + # https://github.com/darshan-hpc/darshan/pull/1070 + # + # However, 5.0.6 and later can cause serialization of file writes + # See https://github.com/open-mpi/ompi/issues/13376 + # To mimize the performance impact of writing the log file, we set + # hint cb_nodes to 1. + # + # Note hint romio_no_indep_rw is an MPI thing, not implemented in + # OpenMPI. + # + __DARSHAN_LOG_HINTS_DEFAULT="cb_nodes=1" + + # if test "x$ompi_ge_5_0_6" = x0 ; then + # __DARSHAN_LOG_HINTS_DEFAULT="romio_no_indep_rw=true;cb_nodes=1" + # else + # __DARSHAN_LOG_HINTS_DEFAULT="romio_no_indep_rw=true;cb_nodes=4" + # fi + fi + __DARSHAN_LOG_HINTS=$__DARSHAN_LOG_HINTS_DEFAULT fi # determine if the MPI library includes MPI-IO functions or not @@ -889,6 +923,10 @@ else is_ompi=0 fi +# __DARSHAN_LOG_HINTS is used by both MPI and non-MPI configurations +AC_DEFINE_UNQUOTED([__DARSHAN_LOG_HINTS], "$__DARSHAN_LOG_HINTS", + [Comma-separated list of MPI-IO hints for log file write]) + AC_SUBST(ENABLE_LD_PRELOAD, ["$enable_ld_preload"]) AC_SUBST(ENABLE_CUSERID, ["$enable_cuserid"]) AC_SUBST(ENABLE_GROUP_READABLE_LOGS, ["$enable_group_readable_logs"]) @@ -923,6 +961,7 @@ AC_SUBST(PNETCDF_PATH, ["$with_pnetcdf"]) AC_SUBST(DAOS_PATH, ["$with_daos"]) AC_SUBST(LDMS_PATH, ["$LDMS_HOME"]) AC_SUBST(HAVE_OPEN_MPI, ["$is_ompi"]) +AC_SUBST(OMPI_GE_5_0_6, ["$ompi_ge_5_0_6"]) AM_CONDITIONAL(ENABLE_MMAP_LOGS, [test "x$enable_mmap_logs" = xyes]) AM_CONDITIONAL(ENABLE_LDPRELOAD, [test "x$enable_ld_preload" = xyes]) diff --git a/darshan-runtime/test/tst_runs.sh b/darshan-runtime/test/tst_runs.sh index 00a98cb9f..63192e4aa 100755 --- a/darshan-runtime/test/tst_runs.sh +++ b/darshan-runtime/test/tst_runs.sh @@ -1,19 +1,11 @@ #!/bin/bash +# Note this script is run during "make check" and "make install" must +# run before "make check". + # Exit immediately if a command exits with a non-zero status. set -e -TODAY_DATE_PATH=`date "+%Y/%-m/%-d"` -TST_DARSHAN_LOG_PATH="${TST_DARSHAN_LOG_PATH}/${TODAY_DATE_PATH}" -mkdir -p ${TST_DARSHAN_LOG_PATH} - -# check what file system is used -echo "df -T ${TST_DARSHAN_LOG_PATH}" -df -T ${TST_DARSHAN_LOG_PATH} - -echo "findmnt -n -o FSTYPE --target ${TST_DARSHAN_LOG_PATH}" -findmnt -n -o FSTYPE --target ${TST_DARSHAN_LOG_PATH} - if test "x$USERNAME_ENV" = xno ; then USERNAME_ENV=$USER fi @@ -31,6 +23,36 @@ else DARSHAN_CONFIG=../../darshan-util/darshan-config fi echo "DARSHAN_CONFIG=$DARSHAN_CONFIG" +echo "" + +TODAY_DATE_PATH=`date "+%Y/%-m/%-d"` +LOG_PATH_ENV=`$DARSHAN_CONFIG --log-path-by-env` +if test "x${LOG_PATH_ENV}" != x ; then + if test "x${!LOG_PATH_ENV}" = x ; then + echo "" + echo "Warning: ---------------------------------------------------------" + echo " Darshan was configure with --log-path-by-env set to $LOG_PATH_ENV" + echo " but this run-time environment variable is currently not set !" + echo " Darshan now uses the path set in configure option of --log-path :" + echo " $TST_DARSHAN_LOG_PATH" + echo "------------------------------------------------------------------" + echo "" + TST_DARSHAN_LOG_PATH="${TST_DARSHAN_LOG_PATH}/${TODAY_DATE_PATH}" + else + TST_DARSHAN_LOG_PATH="${!LOG_PATH_ENV}" + fi +else + TST_DARSHAN_LOG_PATH="${TST_DARSHAN_LOG_PATH}/${TODAY_DATE_PATH}" +fi +echo "TST_DARSHAN_LOG_PATH=$TST_DARSHAN_LOG_PATH" +mkdir -p ${TST_DARSHAN_LOG_PATH} + +# check what file system is used +echo "df -T ${TST_DARSHAN_LOG_PATH}" +df -T ${TST_DARSHAN_LOG_PATH} + +echo "findmnt -n -o FSTYPE --target ${TST_DARSHAN_LOG_PATH}" +findmnt -n -o FSTYPE --target ${TST_DARSHAN_LOG_PATH} $DARSHAN_CONFIG --all