Fio (et al.) postprocessing performance improvements

This change modifies the support in BenchPostprocess.pm to expect
timeseries data in the workload hash to be a hash of hashes, using the
timestamp as the key instead of an array of hashes.  This ensures that
each timestamp is unique, and it makes each timestamp's data accessible
in constant-time rather than requiring a linear (O(n^2)) search.  For
large datasets, this results in a dramatic improvement in execution time
for producing the aggregate results.

Also:
* Do not re-write fio job file in post-process only mode
* Address code formatting and other code quality issues
* Tweak agent/tool-scripts/postprocess unit test script
  * Limit the number of lines in result.json for fio tests
  * Remove the reference-result symlink from the process-iteration-samples-0 gold
* Add unit test for post-process only with no directory
* Add a test which stresses fio-postprocess performance

Co-authored-by: Peter Portante <[email protected]>

Author: webbnh

.gitignore

@@ -3,13 +3,22 @@ __pycache__
 *.swp
 unittests.log
 run-unittests.log
+build
 dist
 *~
 ????-*.patch
+/.env
+.DS_Store
 .npmrc
 .yarnrc
-*.egg-info
+*.egg-info/
 .tox
 .eggs
+pip-wheel-metadata
+.idea/
+.pytest_cache/
+.venv
+.vscode
 Pipfile
 Pipfile.lock
+*,cover

agent/bench-scripts/pbench-fio

@@ -453,7 +453,7 @@ function fio_process_options() {
 				found=1
 				break;
 			fi
-	        done
+		done
 		if [[ ${found} -ne 1 ]]; then
 			error_log "Unsupported --test-types=${test_types} encountered"
 			usage
@@ -564,8 +564,6 @@ function fio_create_jobfile() {
 		error_log "Failed to create job file \"${fio_job_file}\""
 		exit 1
 	fi
-	echo "Created the following job file (${fio_job_file}):"
-	cat ${fio_job_file}
 }
 
 function does_wildcard_exist() {
@@ -809,8 +807,17 @@ function fio_run_benchmark() {
 					fi
 					if [ -e $iteration_dir ]; then
 						fio_job_file="$iteration_dir/fio.job"
-						#             ARG   1            2           3             4          5         6       7          8           9            10           11                         12     13              14
-						fio_create_jobfile "$test_type" "$ioengine" "$block_size" "$iodepth" "$direct" "$sync" "$runtime" "$ramptime" "$file_size" "$rate_iops" "$histogram_interval_msec" "$dev" "$fio_job_file" "$numjobs"
+						local _action=""
+						if [ "$postprocess_only" = "n" ]; then
+							_action="Created"
+							#             ARG   1            2           3             4          5         6       7          8           9            10           11                         12     13              14
+							fio_create_jobfile "$test_type" "$ioengine" "$block_size" "$iodepth" "$direct" "$sync" "$runtime" "$ramptime" "$file_size" "$rate_iops" "$histogram_interval_msec" "$dev" "$fio_job_file" "$numjobs"
+						else
+							_action="Found"
+						fi
+						echo "${_action} the following job file (${fio_job_file}):"
+						cat ${fio_job_file}
+
 						iteration_failed=0
 						sample_failed=0
 						for sample in `seq 1 $nr_samples`; do
@@ -824,16 +831,17 @@ function fio_run_benchmark() {
 								mkdir -p $benchmark_results_dir
 								fio_run_job "$iteration" "$benchmark_results_dir" "$fio_job_file" "$dev" "$clients"
 							else
-								# if we are only postprocessing, then we might have to untar an existing result
-								pushd $iteration_dir >/dev/null
-								if [ ! -e sample$sample ]; then
-									if [ -e sample$sample.tar.xz ]; then
+								# we are only postprocessing, so we might have to untar an existing result
+								if [ ! -e $benchmark_results_dir ]; then
+									if [ -e ${iteration_dir}/sample${sample}.tar.xz ]; then
+										pushd $iteration_dir >/dev/null
 										tar Jxf sample$sample.tar.xz && /bin/rm sample$sample.tar.xz
+										popd >/dev/null
 									else
 										echo "sample $sample missing.  There should be $nr_samples samples"
+										break 2
 									fi
 								fi
-								popd >/dev/null
 							fi
 							debug_log "post-processing fio result"
 							echo "$script_path/postprocess/$benchmark-postprocess \"$benchmark_results_dir\" \"$tool_label_pattern\" \"$tool_group\"" >"$benchmark_results_dir/$benchmark-postprocess.cmd"
@@ -847,7 +855,7 @@ function fio_run_benchmark() {
 							fi
 							if [ $sample_failed -eq 1 ]; then
 								# we need all samples to be good, so bust out of testing samples now
-								break
+								break 2
 							fi
 						done
 					fi
@@ -859,7 +867,7 @@ function fio_run_benchmark() {
 						((failures++))
 					fi
 					if [ $fail -eq 0 -o $failures -ge $max_failures ]; then
-						break
+						break 1
 					fi
 				done
 			let count=$count+1 # now we can move to the next iteration