Add FireSim changes to the standard 2017 branch

README.md

@@ -78,6 +78,18 @@
    Modify the generated "./${CONFIG}-spec-{$INPUT}/run.sh" script as required to 
    run the binaries in your new environment.  
 
+**To run binaries with metrics**
+In addition to directly running the benchmarks, there are also suite-specific
+scripts that collect performance metrics and configure tests. These will be
+available in build/. To run (for example) intspeed for the perl benchmark with
+one thread:
+
+    ./intspeed.sh 600.perlbench_s --threads 1
+
+Three outputs will be available in ~/outputs:
+   - BENCHNAME.out: stdout for the benchmark
+   - BENCHNAME.err: stderr for the benchmark
+   - BENCHNAME.csv: csv-formatted metrics
 
 **TODO**
 

commands/intrate/500.perlbench_r.train.cmd

@@ -0,0 +1,5 @@
+-I./lib diffmail.pl 2 550 15 24 23 100 > diffmail.2.550.15.24.23.100.out 2>> diffmail.2.550.15.24.23.100.err
+-I./lib perfect.pl b 3 > perfect.b.3.out 2>> perfect.b.3.err
+-I. -I./lib scrabbl.pl < scrabbl.in > scrabbl.out 2>> scrabbl.err
+-I./lib splitmail.pl 535 13 25 24 1091 1 > splitmail.535.13.25.24.1091.1.out 2>> splitmail.535.13.25.24.1091.1.err
+-I. -I./lib suns.pl > suns.out 2>> suns.err

commands/intrate/502.gcc_r.train.cmd

@@ -0,0 +1,3 @@
+200.c -O3 -finline-limit=50000 -o 200.opts-O3_-finline-limit_50000.s > 200.opts-O3_-finline-limit_50000.out 2>> 200.opts-O3_-finline-limit_50000.err
+scilab.c -O3 -finline-limit=50000 -o scilab.opts-O3_-finline-limit_50000.s > scilab.opts-O3_-finline-limit_50000.out 2>> scilab.opts-O3_-finline-limit_50000.err
+train01.c -O3 -finline-limit=50000 -o train01.opts-O3_-finline-limit_50000.s > train01.opts-O3_-finline-limit_50000.out 2>> train01.opts-O3_-finline-limit_50000.err

commands/intrate/505.mcf_r.train.cmd

@@ -0,0 +1 @@
+inp.in  > inp.out 2>> inp.err

commands/intrate/520.omnetpp_r.train.cmd

@@ -0,0 +1 @@
+-c General -r 0 > omnetpp.General-0.out 2>> omnetpp.General-0.err

commands/intrate/523.xalancbmk_r.train.cmd

@@ -0,0 +1 @@
+-v allbooks.xml xalanc.xsl > train-allbooks.out 2>> train-allbooks.err

commands/intrate/525.x264_r.train.cmd

@@ -0,0 +1 @@
+--dumpyuv 50 --frames 142 -o BuckBunny_New.264 BuckBunny.yuv 1280x720 > run_000-142_x264_r_base.host-m64_x264.out 2>> run_000-142_x264_r_base.host-m64_x264.err

commands/intrate/531.deepsjeng_r.train.cmd

@@ -0,0 +1 @@
+train.txt > train.out 2>> train.err

commands/intrate/541.leela_r.train.cmd

@@ -0,0 +1 @@
+train.sgf > train.out 2>> train.err

commands/intrate/548.exchange2_r.train.cmd

@@ -0,0 +1 @@
+1 > exchange2.txt 2>> exchange2.err

commands/intrate/557.xz_r.train.cmd

@@ -0,0 +1,2 @@
+input.combined.xz 40 a841f68f38572a49d86226b7ff5baeb31bd19dc637a922a972b2e6d1257a890f6a544ecab967c313e370478c74f760eb229d4eef8a8d2836d233d3e9dd1430bf 6356684 -1 8 > input.combined-40-8.out 2>> input.combined-40-8.err
+IMG_2560.cr2.xz 40 ec03e53b02deae89b6650f1de4bed76a012366fb3d4bdc791e8633d1a5964e03004523752ab008eff0d9e693689c53056533a05fc4b277f0086544c6c3cbbbf6 40822692 40824404 4 > IMG_2560.cr2-40-4.out 2>> IMG_2560.cr2-40-4.err

commands/intspeed/600.perlbench_s.train.cmd

@@ -0,0 +1,5 @@
+-I./lib diffmail.pl 2 550 15 24 23 100 > diffmail.2.550.15.24.23.100.out 2>> diffmail.2.550.15.24.23.100.err
+-I./lib perfect.pl b 3 > perfect.b.3.out 2>> perfect.b.3.err
+-I. -I./lib scrabbl.pl < scrabbl.in > scrabbl.out 2>> scrabbl.err
+-I./lib splitmail.pl 535 13 25 24 1091 1 > splitmail.535.13.25.24.1091.1.out 2>> splitmail.535.13.25.24.1091.1.err
+-I. -I./lib suns.pl > suns.out 2>> suns.err

commands/intspeed/602.gcc_s.train.cmd

@@ -0,0 +1,3 @@
+200.c -O3 -finline-limit=50000 -o 200.opts-O3_-finline-limit_50000.s > 200.opts-O3_-finline-limit_50000.out 2>> 200.opts-O3_-finline-limit_50000.err
+scilab.c -O3 -finline-limit=50000 -o scilab.opts-O3_-finline-limit_50000.s > scilab.opts-O3_-finline-limit_50000.out 2>> scilab.opts-O3_-finline-limit_50000.err
+train01.c -O3 -finline-limit=50000 -o train01.opts-O3_-finline-limit_50000.s > train01.opts-O3_-finline-limit_50000.out 2>> train01.opts-O3_-finline-limit_50000.err

commands/intspeed/605.mcf_s.train.cmd

@@ -0,0 +1 @@
+inp.in  > inp.out 2>> inp.err

commands/intspeed/620.omnetpp_s.train.cmd

@@ -0,0 +1 @@
+-c General -r 0 > omnetpp.General-0.out 2>> omnetpp.General-0.err

commands/intspeed/623.xalancbmk_s.train.cmd

@@ -0,0 +1 @@
+-v allbooks.xml xalanc.xsl > train-allbooks.out 2>> train-allbooks.err

commands/intspeed/625.x264_s.train.cmd

@@ -0,0 +1 @@
+--dumpyuv 50 --frames 142 -o BuckBunny_New.264 BuckBunny.yuv 1280x720 > run_000-142_x264_s_base.host-m64_x264.out 2>> run_000-142_x264_s_base.host-m64_x264.err

commands/intspeed/631.deepsjeng_s.train.cmd

@@ -0,0 +1 @@
+train.txt > train.out 2>> train.err

commands/intspeed/641.leela_s.train.cmd

@@ -0,0 +1 @@
+train.sgf > train.out 2>> train.err

commands/intspeed/648.exchange2_s.train.cmd

@@ -0,0 +1 @@
+1 > exchange2.txt 2>> exchange2.err

commands/intspeed/657.xz_s.train.cmd

@@ -0,0 +1,2 @@
+input.combined.xz 40 a841f68f38572a49d86226b7ff5baeb31bd19dc637a922a972b2e6d1257a890f6a544ecab967c313e370478c74f760eb229d4eef8a8d2836d233d3e9dd1430bf 6356684 -1 8 > input.combined-40-8.out 2>> input.combined-40-8.err
+IMG_2560.cr2.xz 40 ec03e53b02deae89b6650f1de4bed76a012366fb3d4bdc791e8633d1a5964e03004523752ab008eff0d9e693689c53056533a05fc4b277f0086544c6c3cbbbf6 40822692 40824404 4 > IMG_2560.cr2-40-4.out 2>> IMG_2560.cr2-40-4.err

gen_binaries.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-#set -e
+set -e
 
 #############
 # TODO
@@ -24,7 +24,7 @@ REDIRECT=false
 # CML arguments
 # idiomatic parameter and option handling in sh
 compileFlag=false
-genCommands=false
+genCommandsFlag=false
 
 # intrate, fprate, intspeed, fpspeed
 # Supersets spec{speed,rate}, and all, are not supported
@@ -53,7 +53,7 @@ do
             ;;
         --input)
             shift;
-            inputs_type=$1
+            input_type=$1
             ;;
         -h | -H | -help)
             usage
@@ -114,7 +114,7 @@ if [ "$compileFlag" = true ]; then
       --action runsetup ${suite_type} > ${build_dir}/${H_CONFIG}-${suite_type}-build.log
 
    for b in ${benchmarks[@]}; do
-      output_dir=${overlay_dir}/${suite_type}/$b
+      output_dir=${overlay_dir}/${suite_type}/${input_type}/$b
       mkdir -p $output_dir
       bmark_base_dir=$SPEC_DIR/benchspec/CPU/$b
       unprefixed=${b:4}
@@ -123,7 +123,7 @@ if [ "$compileFlag" = true ]; then
       if [[ "${input_type}" == "ref" ]]; then
          host_bmk_dir=${bmark_base_dir}/run/run_base_ref${class}_${H_CONFIG}-m64.0000;
       else
-         host_bmk_dir=${bmark_base_dir}/run/run_base_${input}_${H_CONFIG}-m64.0000;
+         host_bmk_dir=${bmark_base_dir}/run/run_base_${input_type}_${H_CONFIG}-m64.0000;
       fi
 
       # Copy the inputs from the host build
@@ -145,7 +145,7 @@ if [ "$compileFlag" = true ]; then
       echo "#!/bin/bash" > ${run_script}
       echo "#This script was generated by Speckle gen_binaries.sh" >> ${run_script}
 
-      IFS=$'\n' read -d '' -r -a commands < $build_dir/../commands/$suite_type/${b}.${input_type}.cmd
+      IFS=$'\n' read -d '' -r -a commands < $build_dir/../commands/$suite_type/${b}.${input_type}.cmd || [ "${commands[0]}" ]
       workload_idx=0
       for input in "${commands[@]}"; do
          if [[ ${input:0:1} != '#' ]]; then # allow us to comment out lines in the cmd files
@@ -161,11 +161,14 @@ if [ "$compileFlag" = true ]; then
             echo "$cmd" >> ${run_script}
             echo "$cmd" >> ${workload_run_script}
             chmod +x $workload_run_script
-            ((workload_idx++))
+            workload_idx="$((workload_idx+1))"
          fi
       done
       chmod +x $run_script
    done
+   # Copy the master runscript into the overlay directory
+   cp ${build_dir}/../spec17-run-scripts/${suite_type}.sh ${overlay_dir}/${suite_type}/${input_type}
+
 fi
 
 # Produces the .cmd files for a benchmark suite
@@ -189,4 +192,3 @@ fi
 
 echo ""
 echo "Done!"
-

host.cfg

@@ -211,7 +211,7 @@ default:               # data model applies to all benchmarks
 #                   switches here.  See also 'About the -fno switches' below.
 #
 default=base:         # flags for all base 
-   OPTIMIZE       = -g -O3 -march=native -fno-unsafe-math-optimizations  -fno-tree-loop-vectorize
+   OPTIMIZE       = -g -O3 -march=native -fno-unsafe-math-optimizations
 
 intrate,intspeed=base: # flags for integer base
 %if %{bits} == 32                    
@@ -272,7 +272,7 @@ default=peak:
 
 #--------- EDIT to match your version -----------------------------------------
 default:
-   sw_compiler001   = C/C++/Fortran: Version 6.2.0 of GCC, the
+   sw_compiler001   = C/C++/Fortran: Version 4.8.5 of GCC, the
    sw_compiler002   = GNU Compiler Collection
 
 #--------- EDIT info about you ------------------------------------------------

spec17-run-scripts/intrate.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+#defaults TODO: Make = number of harts?
+copies=1
+counters=0
+
+function usage
+{
+    echo "usage: intrate.sh <benchmark-name> [-H | -h | --help] [--copies <int>] [--workload <int>] [--counters]"
+    echo "   benchmark-name: the spec17 run directory with binary and inputs"
+    echo "   copies: number of rate instances to run (2GiB each) Default: ${copies}"
+    echo "   counters: if set, runs an hpm_counters instance on each hart"
+}
+
+if [ $# -eq 0 -o "$1" == "--help" -o "$1" == "-h" -o "$1" == "-H" ]; then
+    usage
+    exit 3
+fi
+
+bmark_name=$1
+shift
+
+while test $# -gt 0
+do
+   case "$1" in
+        --copies)
+            shift;
+            copies=$1
+            ;;
+        --counters)
+            counters=1;
+            ;;
+        -h | -H | --help)
+            usage
+            exit
+            ;;
+        --*) echo "ERROR: bad option $1"
+            usage
+            exit 1
+            ;;
+        *) echo "ERROR: bad argument $1"
+            usage
+            exit 2
+            ;;
+    esac
+    shift
+done
+
+mkdir -p ~/output
+work_dir=$PWD
+
+runscript="run.sh"
+echo "Starting rate $bmark_name run with $copies copies"
+
+echo "Creating run directories"
+for i in `seq 0 $[ ${copies} - 1 ]`; do
+    cp -al $work_dir/$bmark_name ${work_dir}/copy-$i
+done
+
+# In some systems we might not support for our counter program; so optionally disable it 
+if [ -z "$DISABLE_COUNTERS" -a "$counters" -ne 0 ]; then
+    start_counters
+fi
+
+for i in `seq 0 $[ ${copies} - 1 ]`; do
+    cd $work_dir/copy-$i
+    echo "name,RealTime,UserTime,KernelTime,copy" >> ~/output/${bmark_name}_${i}.csv
+    /usr/bin/time -a -o ~/output/${bmark_name}_${i}.csv -f "${bmark_name},%e,%U,%S,${i}" \
+       ./run.sh > ~/output/${bmark_name}_${i}.out 2> ~/output/${bmark_name}_${i}.err &
+done
+sleep 10
+while pgrep -f run.sh > /dev/null; do sleep 10; done
+
+if [ -z "$DISABLE_COUNTERS" -a "$counters" -ne 0 ]; then
+    stop_counters
+fi

spec17-run-scripts/intspeed.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Defaults
+num_threads=1
+counters=0
+
+function usage
+{
+    echo "usage: intspeed.sh <benchmark-name> [-H | -h | --help] [--threads <int>] [--workload <int>]"
+    echo "   benchmark-name: the spec17 run directory with binary and inputs"
+    echo "   threads: number of OpenMP threads to use. Default: ${num_threads}"
+    echo "   workload: which workload number to run. Leaving this unset runs all."
+    echo "   counters: if set, runs an hpm_counters instance on each hart"
+}
+
+if [ $# -eq 0 -o "$1" == "--help" -o "$1" == "-h" -o "$1" == "-H" ]; then
+    usage
+    exit 3
+fi
+
+bmark_name=$1
+shift
+
+while test $# -gt 0
+do
+   case "$1" in
+        --workload)
+            shift;
+            workload_num=$1
+            ;;
+        --threads)
+            shift;
+            num_threads=$1
+            ;;
+        --counters)
+            counters=1;
+            ;;
+        -h | -H | -help)
+            usage
+            exit
+            ;;
+        --*) echo "ERROR: bad option $1"
+            usage
+            exit 1
+            ;;
+        *) echo "ERROR: bad argument $1"
+            usage
+            exit 2
+            ;;
+    esac
+    shift
+done
+
+work_dir=$PWD
+export OMP_NUM_THREADS=$num_threads
+mkdir -p ~/output
+
+if [ -z "$workload_num" ]; then
+    runscript="run.sh"
+    echo "Starting speed $bmark_name run with $OMP_NUM_THREADS threads"
+else
+    runscript="run_workload${workload_num}.sh"
+    echo "Starting speed $bmark_name (workload ${workload_num}) run with $OMP_NUM_THREADS threads"
+fi
+
+# In some systems we might not support for our counter program; so optionally disable it 
+if [ -z "$DISABLE_COUNTERS" -a "$counters" -ne 0 ]; then
+    start_counters
+fi
+
+# Actually start the workload
+cd $work_dir/${bmark_name}
+
+if [ -z "$workload_num" ]; then
+    full_name=${bmark_name}
+else
+    full_name=${bmark_name}_${workload_num}
+fi
+
+# busybox has a bug in time where escape characters (e.g. \n) are not
+# interpreted correctly, we have to put the CSV header in manually
+echo "name,RealTime,UserTime,KernelTime" >> ~/output/${full_name}.csv
+/usr/bin/time -a -o ~/output/${full_name}.csv -f "${full_name},%e,%U,%S" \
+    ./${runscript} > ~/output/${full_name}.out 1> ~/output/${full_name}.err
+
+if [ -z "$DISABLE_COUNTERS" -a "$counters" -ne 0 ]; then
+    stop_counters
+fi