Add local and remote pre-check for linpack

This is a back-port of commit b51b116 (PR #2856) from `main`.

We add a local and remote pre-check function for linpack.

We now wait at most 60 seconds for the linpack process to start, and
then we wait for the PID to exit instead of a file to exist.  That way
if a file is never written that we expect, but the PID has died, we'll
not wait forever.

Further, we move the "version" check (of sorts) to the linpack driver
script to allow for the remote check to work without having the code in
two places.

Author: portante

agent/bench-scripts/driver/linpack

@@ -10,6 +10,7 @@ linpack_input_file_name="linpack.input"
 linpack_metadata_file_name="linpack.meta"
 linpack_output_file_name="linpack.out"
 linpack_command_file_name="linpack.cmd"
+linpack_pid_file_name="linpack.pid"
 
 binary=""
 
@@ -38,15 +39,19 @@ kmp_affinity_args_def="nowarnings,compact,1,0,granularity=fine"
 kmp_affinity_args=""
 numactl_args=""
 
-# Output directory required.
+# Output directory, defaults to the current working directory if not provided.
 output_dir=""
 
+# The linpack driver always performs pre-checks. When requested, the driver will
+# only run the pre-check and exit.
+pre_check_only=0
+
 function usage() {
 	cat <<-__EOF__
 Usage: ${script_name} [--problem-sizes=#[[,#]...]] [--leading-dimenions=#[[,#]...]]
         [--run-trials=#[[,#]]...] [--alignment-values=#[[,#]...]] [--use-omp={y|n}]
         [--threads=#] [--kmp-affinity=<options>] [--numactl-args=<args>]
-        [--output-dir=<directory>] <linpack binary path>
+        [--output-dir=<directory>] <linpack version number installed>
 
 Optional output control:
 
@@ -128,7 +133,7 @@ an invocation errors.
 }
 
 # Process options and arguments
-opts=$(getopt -q -o h --longoptions "header:,subheader:,problem-sizes:,leading-dimensions:,alignment-values:,run-trials:,threads:,use-omp:,kmp-affinity:,numactl-args:,output-dir:,help" -n "${script_name}" -- "${@}")
+opts=$(getopt -q -o h --longoptions "header:,subheader:,problem-sizes:,leading-dimensions:,alignment-values:,run-trials:,threads:,use-omp:,kmp-affinity:,numactl-args:,output-dir:,pre-check-only,help" -n "${script_name}" -- "${@}")
 if [[ ${?} -ne 0 ]]; then
 	printf -- "%s %s\n\n\tunrecognized option specified\n\n" "${script_name}" "${*}" >&2
 	usage >&2
@@ -211,6 +216,9 @@ while true; do
 			shift
 		fi
 		;;
+	--pre-check-only)
+		pre_check_only=1
+		;;
 	-h|--help)
 		help
 		exit 0
@@ -225,16 +233,26 @@ while true; do
 	esac
 done
 
-binary=${1}
-if [[ -z "${binary}" ]]; then
-	printf -- "[%s] ERROR: You must specify the location of the LINPACK binary\n\n" "${script_name}" >&2
+ver=${1}
+if [[ -z "${ver}" ]]; then
+	printf -- "[%s] ERROR: You must specify the version of the LINPACK binary\n\n" "${script_name}" >&2
 	usage >&2
 	exit 2
 fi
+
+# Installation directory, optional 2nd argument, defaults to /usr/local.
+install_prefix_dir=${2:-"/usr/local"}
+
+binary="${install_prefix_dir}/pbench-linpack-${ver}/benchmarks/linpack/xlinpack_xeon64"
 if [[ ! -x "${binary}" ]]; then
-	printf -- "[%s] ERROR: The --binary must exist and be executable\n" "${script_name}" >&2
+	printf -- "[%s] ERROR: The --binary, '${binary}', must exist and be executable\n" "${script_name}" >&2
 	exit 2
 fi
+
+if [[ ${pre_check_only} -ne 0 ]]; then
+	exit 0
+fi
+
 output_dir=${output_dir:-$(pwd)}
 if [[ ! -d "${output_dir}" ]]; then
 	printf -- "[%s] ERROR: Specified --output-dir '${output_dir}' is not a directory\n" "${script_name}" >&2
@@ -270,6 +288,7 @@ linpack_input_file="${output_dir}/${linpack_input_file_name}"
 linpack_metadata_file="${output_dir}/${linpack_metadata_file_name}"
 linpack_output_file="${output_dir}/${linpack_output_file_name}"
 linpack_command_file="${output_dir}/${linpack_command_file_name}"
+linpack_pid_file="${output_dir}/${linpack_pid_file_name}"
 
 # N.B. - Trailing whitespace is required.
 numactl_cmd=${numactl_args:+"numactl ${numactl_args} "}
@@ -312,6 +331,9 @@ cat > ${linpack_input_file} <<-__EOF__
 	${alignment_values} # alingment values (in KBytes)
 	__EOF__
 
+# Declare ourselves to anybody waiting.
+echo "$$" > ${linpack_pid_file}
+
 # Now we can execute the final LINPACK command.
 source ${linpack_command_file}
 exit_code=${?}

agent/bench-scripts/driver/linpack-wait

@@ -3,7 +3,30 @@
 
 # A very simple method to wait for the linpack driver to report it has
 # finished.
-while [[ ! -e "${1}/linpack.meta" ]]; do
+
+if [[ ! -d "${1}" ]]; then
+	printf -- "linpack-wait: target directory, '${1}', does not exist!\n" >&2
+	exit 1
+fi
+
+# Wait for the linpack pid file to show up.
+let cnt=60
+while [[ ${cnt} -gt 0 && -d "${1}" && ! -e "${1}/linpack.pid" ]]; do
+	sleep 1
+	(( cnt-- ))
+done
+if [[ ! -d "${1}" ]]; then
+	printf -- "linpack-wait: target directory, '${1}', no longer exists!\n" >&2
+	exit 1
+fi
+if [[ ! -e "${1}/linpack.pid" ]]; then
+	printf -- "linpack-wait: linpack pid file, '${1}/linpack.pid', failed to show up\n" >&2
+	exit 1
+fi
+
+# At this point, we wait for the linpack process to stop running.
+pid=$(< "${1}/linpack.pid")
+while [[ -d /proc/${pid} ]]; do
 	sleep 1
 done
 exit 0

agent/bench-scripts/pbench-linpack

@@ -27,6 +27,11 @@ pbench_bin="$(realpath -e ${script_path}/..)"
 export benchmark="linpack"
 
 # Defaults
+
+# This script always runs a pre-check. When non-local clients are used, as part
+# of the pre-check operation, it invokes itself remotely to ONLY run a local
+# pre-check and exit.
+pre_check_only=0
 def_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
 threads=${def_threads}
 def_nr_samples=2
@@ -49,7 +54,7 @@ function usage {
 }
 
 # Process options and arguments
-opts=$(getopt -q -o C:c:h --longoptions "config:,clients:,samples:,threads:,tool-group:,sysinfo:,help" -n "getopt.sh" -- "${@}")
+opts=$(getopt -q -o C:c:h --longoptions "config:,clients:,help,pre-check-only,samples:,sysinfo:,threads:,tool-group:" -n "getopt.sh" -- "${@}")
 if [[ ${?} -ne 0 ]]; then
 	printf -- "%s %s\n\n\tunrecognized option specified\n\n" "${script_name}" "${*}" >&2
 	usage >&2
@@ -72,27 +77,30 @@ while true; do
 			shift
 		fi
 		;;
+	--pre-check-only)
+		pre_check_only=1
+		;;
 	--samples)
 		if [[ -n "${1}" ]]; then
 			nr_samples="${1}"
 			shift
 		fi
 		;;
-	--threads)
+	--sysinfo)
 		if [[ -n "${1}" ]]; then
-			threads="${1}"
+			sysinfo="${1}"
 			shift
 		fi
 		;;
-	--tool-group)
+	--threads)
 		if [[ -n "${1}" ]]; then
-			tool_group="${1}"
+			threads="${1}"
 			shift
 		fi
 		;;
-	--sysinfo)
+	--tool-group)
 		if [[ -n "${1}" ]]; then
-			sysinfo="${1}"
+			tool_group="${1}"
 			shift
 		fi
 		;;
@@ -113,24 +121,49 @@ while true; do
 done
 verify_common_bench_script_options ${tool_group} ${sysinfo}
 
-ver="$(pbench-config version ${benchmark})"
-if [[ -z "${ver}" ]]; then
-	error_log "${script_name}: package version is missing in config file"
-	exit 1
-fi
-if [[ -z "${linpack_dir}" ]]; then
-	linpack_dir="/usr/local/${script_name}-${ver}/benchmarks/linpack"
-	linpack_dir_kind="default"
-else
-	linpack_dir_kind="provided"
+function pre_check {
+	# Invoke the linpack driver to perform a pre-check that it will be able to
+	# execute the benchmark. The expected version is the first argument, and the
+	# second is the expected directory prefix for the linpack installation.
+	local ver=${1}
+	local install_prefix_arg=${2}
+
+	${pbench_bin}/bench-scripts/driver/linpack --pre-check-only ${ver} ${install_prefix_arg}
+	return ${?}
+}
+
+if [[ ${pre_check_only} -ne 0 ]]; then
+	# We have been invoked remotely to check for the expected version of
+	# linpack installed.  The remote invocation has provided the arguments to
+	# pass to the pre-check function (see --pre-check-only invocation below).
+	pre_check ${@}
+	exit ${?}
 fi
-if [[ ! -d "${linpack_dir}" ]]; then
-	error_log "${script_name}: the ${linpack_dir_kind} linpack directory, ${linpack_dir}, does not exist"
+
+linpack_ver="$(pbench-config version ${benchmark})"
+if [[ -z "${linpack_ver}" ]]; then
+	error_log "${script_name}: package version is missing in config file"
 	exit 1
 fi
-linpack_cmd="${linpack_dir}/xlinpack_xeon64"
-if [[ ! -x "${linpack_cmd}" ]]; then
-	error_log "${script_name}: the expected linpack command, ${linpack_cmd}, does not exist"
+
+# Run the pre-check.
+let not_found=0
+for client in ${clients//,/ }; do
+	if pbench-is-local "${client}"; then
+		pre_check ${linpack_ver} ${PBENCH_LINPACK_INSTALL_PREFIX_DIR}
+		if [[ ${?} -ne 0 ]]; then
+			error_log "${script_name}: linpack not installed locally"
+			(( not_found++ ))
+		fi
+	else
+		ssh ${ssh_opts} ${client} ${script_name} --pre-check-only ${linpack_ver} ${PBENCH_LINPACK_INSTALL_PREFIX_DIR}
+		if [[ ${?} -ne 0 ]]; then
+			error_log "${script_name}: linpack not installed on client ${client}"
+			(( not_found++ ))
+		fi
+	fi
+done
+if [[ ${not_found} -gt 0 ]]; then
 	exit 1
 fi
 
@@ -243,7 +276,8 @@ for thread in ${threads//,/ }; do
 		pbench-start-tools --group=${tool_group} --dir="${sample_dir}"
 
 		run_it="pbench-linpack ${pbench_bin}/bench-scripts/driver/linpack"
-		run_it+=" --output-dir=${sample_dir} --threads=${thread} ${linpack_cmd}"
+		run_it+=" --output-dir=${sample_dir} --threads=${thread}"
+		run_it+=" ${linpack_ver} ${PBENCH_LINPACK_INSTALL_PREFIX_DIR}"
 		screen_it="screen -dmS ${run_it}"
 		for client in ${clients//,/ }; do
 			if pbench-is-local "${client}"; then

agent/bench-scripts/tests/pbench-linpack/test-63.txt

@@ -1,5 +1,6 @@
 +++ Running test-63 pbench-linpack
-[error][1900-01-01T00:00:00.000000] pbench-linpack: the default linpack directory, /usr/local/pbench-linpack-11.1.3/benchmarks/linpack, does not exist
+[linpack] ERROR: The --binary, '/usr/local/pbench-linpack-11.1.3/benchmarks/linpack/xlinpack_xeon64', must exist and be executable
+[error][1900-01-01T00:00:00.000000] pbench-linpack: linpack not installed locally
 --- Finished test-63 pbench-linpack (status=1)
 +++ pbench tree state
 /var/tmp/pbench-test-bench/pbench-agent
@@ -12,5 +13,5 @@
 /var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar
 --- pbench tree state
 +++ pbench.log file contents
-[error][1900-01-01T00:00:00.000000] pbench-linpack: the default linpack directory, /usr/local/pbench-linpack-11.1.3/benchmarks/linpack, does not exist
+[error][1900-01-01T00:00:00.000000] pbench-linpack: linpack not installed locally
 --- pbench.log file contents

agent/bench-scripts/tests/pbench-linpack/test-64.env

@@ -1 +1 @@
-export linpack_dir=${_testtmp}/linpack
+export PBENCH_LINPACK_INSTALL_PREFIX_DIR=${_testtmp}

agent/bench-scripts/tests/pbench-linpack/test-64.pre

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 # Setup a fake install of linpack but without the expected executable.
-_linpack_dir=${_testtmp}/linpack
+_linpack_dir=${_testtmp}/pbench-linpack-11.1.3
 mkdir ${_linpack_dir} || exit 1
 exit 0

agent/bench-scripts/tests/pbench-linpack/test-64.txt

@@ -1,5 +1,6 @@
 +++ Running test-64 pbench-linpack
-[error][1900-01-01T00:00:00.000000] pbench-linpack: the expected linpack command, /var/tmp/pbench-test-bench/tmp/linpack/xlinpack_xeon64, does not exist
+[linpack] ERROR: The --binary, '/var/tmp/pbench-test-bench/tmp/pbench-linpack-11.1.3/benchmarks/linpack/xlinpack_xeon64', must exist and be executable
+[error][1900-01-01T00:00:00.000000] pbench-linpack: linpack not installed locally
 --- Finished test-64 pbench-linpack (status=1)
 +++ pbench tree state
 /var/tmp/pbench-test-bench/pbench-agent
@@ -12,5 +13,5 @@
 /var/tmp/pbench-test-bench/pbench-agent/tools-v1-default/testhost.example.com/sar
 --- pbench tree state
 +++ pbench.log file contents
-[error][1900-01-01T00:00:00.000000] pbench-linpack: the expected linpack command, /var/tmp/pbench-test-bench/tmp/linpack/xlinpack_xeon64, does not exist
+[error][1900-01-01T00:00:00.000000] pbench-linpack: linpack not installed locally
 --- pbench.log file contents

agent/bench-scripts/tests/pbench-linpack/test-65.pre

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 # Setup a fake install of linpack but with a stub for the expected executable.
-_linpack_dir=${_testtmp}/linpack
-mkdir ${_linpack_dir} || exit 1
+_linpack_dir=${_testtmp}/pbench-linpack-11.1.3/benchmarks/linpack
+mkdir -p ${_linpack_dir} || exit 1
 printf -- "#!/bin/bash\nexit 0\n" > ${_linpack_dir}/xlinpack_xeon64 || exit 1
 chmod 775 ${_linpack_dir}/xlinpack_xeon64 || exit 1
 exit 0