lkmc v2-rc

Unsquashed version at v2-rc-unsquashed, but that cannot be merged as it
breaks bisects at several points. All bugs will not bisect to this
humongous change.

It all started with a conversion of the Bash scripts to Python, mainly
because I couldn't stand not being able to properly use --options for
run which has a million options.

Then since that required a full testing, I decided to do all the
refactorings that I had in mind at once, and so I did and it became
v2-rc.

This is the largest patch I have ever done! OMG a few weeks of extra time.
I'm never writing a Bash script for anything that starts getting big again.

Some of the features are:

* separate build-qemu and build-gem5 commands
* common: convert scripts to python. Add --option for everything
* rename build to build-buildroot now that we are splitting all the build
  commands, Linux kernel to follow
* move all git submodules to submodules/ and all buildroot packages to
  packages/
* refactor the out/ structure. Keep projects on toplevel, because guest
  projects separate archs and host ones don't, making a toplevel arch wrong
* do-release: rename to just release
  https://stackoverflow.com/questions/16174992/cant-get-argparse-to-read-quoted-string-with-dashes-in-it
* run: add --terminal and explain gem5 pdb
* just track the lvimrc
* store CLI kernel config fragment inside buildlroot to avoid conflicts
* gem5: document m5 initparam
* readme: make a bunch of things awesomer
* readme: fix broken refs
* parsec-benchmark: update to 75d55ac446a43c47efb1044844a108c6c330184c
  Could not fetch otherwise.
* gem5: M5_OVERRIDE_PY_SOURCE

Author: cirosantilli

.gitignore

@@ -11,3 +11,7 @@ gitignore*
 /images-*.zip
 /out
 /out.*
+
+# Python trash.
+*.pyc
+__pycache__

.gitmodules

@@ -1,16 +1,20 @@
-[submodule "buildroot"]
-	path = buildroot
+[submodule "submodules/buildroot"]
+	path = submodules/buildroot
 	url = https://github.com/cirosantilli/buildroot
 	ignore = dirty
-[submodule "qemu"]
-	path = qemu
+[submodule "submodules/qemu"]
+	path = submodules/qemu
 	url = https://github.com/cirosantilli/qemu
-[submodule "linux"]
-	path = linux
+# The true upstream does not accept git submodule update --init --depth 1
+# git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
+# But git clone --branch --depth 1 worked weirdly:
+# https://unix.stackexchange.com/questions/338578/linux-kernel-source-code-size-difference
+[submodule "submodules/linux"]
+	path = submodules/linux
 	url = https://github.com/cirosantilli/linux
-[submodule "gem5/gem5"]
-	path = gem5/gem5
+[submodule "submodules/gem5"]
+	path = submodules/gem5
 	url = https://gem5.googlesource.com/public/gem5
-[submodule "parsec-benchmark/parsec-benchmark"]
-	path = parsec-benchmark/parsec-benchmark
+[submodule "submodules/parsec-benchmark"]
+	path = submodules/parsec-benchmark
 	url = https://github.com/cirosantilli/parsec-benchmark

.lvimrc

@@ -0,0 +1,9 @@
+" Use this automatically with:
+"
+" Plugin 'embear/vim-localvimrc'
+" let g:localvimrc_ask = 0
+" let g:localvimrc_sandbox = 0
+
+if &filetype ==# 'c' || &filetype ==# 'cpp'
+  setlocal noexpandtab
+endif

.travis.yml

@@ -21,5 +21,6 @@ script: |
   # The following packages have unmet dependencies:
   # libsdl2-dev : Depends: libegl1-mesa-dev
   #            Depends: libgles2-mesa-dev
+  bash -x ./build-qemu -j 16 -S |& awk 'NR % 1000 == 0'
   bash -x ./build -j 16 -S |& awk 'NR % 1000 == 0'
   bash -x ./run -e 'init=/poweroff.out'

README.adoc

@@ -1,28 +1,26 @@
 #!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
-set -x
-bench_build=false
-bench_buildroot_baseline=false
+set -eux
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+bench_all=false
+bench_buildroot_build=false
+bench_buildroot_baseline_build=false
 bench_gem5_build=false
 bench_linux_boot=false
 default_arch=x86_64
 update_repo=false
 while getopts Aa:Bbglu OPT; do
   case "$OPT" in
     A)
-      bench_build=true
-      bench_buildroot_baseline=true
-      bench_gem5_build=true
-      bench_linux_boot=true
+      bench_all=true
       ;;
     a)
       default_arch="$OPTARG"
       ;;
     b)
-      bench_build=true
+      bench_buildroot_build=true
       ;;
     B)
-      bench_buildroot_baseline=true
+      bench_buildroot_baseline_build=true
       ;;
     g)
       bench_gem5_build=true
@@ -40,9 +38,24 @@ while getopts Aa:Bbglu OPT; do
 done
 shift "$(($OPTIND - 1))"
 comment="${1:-}"
+if \
+  ! "$bench_buildroot_build" && \
+  ! "$bench_buildroot_baseline_build" && \
+  ! "$bench_gem5_build" && \
+  ! "$bench_linux_boot" \
+; then
+  bench_all=true
+fi
+if "$bench_all"; then
+  bench_buildroot_build=true
+  bench_buildroot_baseline_build=true
+  bench_gem5_build=true
+  bench_linux_boot=true
+fi
+getvar="${root_dir}/getvar"
 
 # Create output directory.
-benchmark_repo="${common_root_dir}/../linux-kernel-module-cheat-regression"
+benchmark_repo="${root_dir}/../linux-kernel-module-cheat-regression"
 mkdir -p "$benchmark_repo"
 last_dir="$(ls "$benchmark_repo" | grep -E '^[0-9]' | tail -n 1)"
 if [ -n "$last_dir" ]; then
@@ -51,62 +64,61 @@ else
   seq_id=0
 fi
 seq_id="$(printf '%0.4d' "$seq_id")"
-dir_basename="${seq_id}_${common_sha}"
+dir_basename="${seq_id}_$("$getvar" sha)"
 new_dir="${benchmark_repo}/${dir_basename}"
 mkdir "$new_dir"
 
-if "$bench_build"; then
-  common_arch="$default_arch"
-  common_suffix=bench
-  common_setup
-  rm -rf "$common_out_arch_dir"
-  ./build -a "$common_arch" -B 'BR2_CCACHE=n' -s "$common_suffix"
-  cp "${common_build_dir}/build-time.log" "${new_dir}/build-time-${common_arch}.log"
-  rm -rf "$common_out_arch_dir"
+do_bench_buildroot_build() (
+  arch="$default_arch"
+  build_id=bench
+  if [ "${1:-}" = baseline ]; then
+    baseline=--baseline
+    baseline_suffix=-baseline
+  else
+    baseline=
+    baseline_suffix=
+  fi
+  common_build_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" build_dir)"
+  common_images_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" images_dir)"
+  "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --clean
+  "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --no-all -- source
+  "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id"
+  cp "${common_build_dir}/build-time.log" "${new_dir}/buildroot-build-time-${baseline_suffix}${arch}.log"
+  wc -c "${images_dir}/"* > "${new_dir}/buildroot-image-size-${baseline_suffix}${arch}.log"
+  "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --clean
+)
+
+if "$bench_buildroot_build"; then
+  do_bench_buildroot_build
 fi
 
-if "$bench_buildroot_baseline"; then
-  cd "${common_root_dir}/buildroot"
-  git clean -xdf
-  make "qemu_${default_arch}_defconfig"
-  printf '
-BR2_CCACHE=y
-BR2_TARGET_ROOTFS_CPIO=y
-BR2_TARGET_ROOTFS_EXT2=n
-' >>.config
-  make olddefconfig
-  make source
-  time env -u LD_LIBRARY_PATH make BR2_JLEVEL="$(nproc)"
-  cp output/build/build-time.log "${new_dir}/baseline-build-time-${default_arch}.log"
-  wc -c output/images/* > "${new_dir}/baseline-image-size-${default_arch}.log"
-  git clean -xdf
+if "$bench_buildroot_baseline_build"; then
+  do_bench_buildroot_build baseline
 fi
 
 if "$bench_gem5_build"; then
-  arches='x86_64 arm'
-  for common_arch in $arches; do
-    common_setup
-    cd "${common_gem5_src_dir}"
-    git clean -xdf
-    results_file="${common_gem5_out_dir}/bench-build.txt"
-    gem5_outdir="${common_out_dir}/bench_build"
-    rm -fr "$results_file" "${gem5_outdir}"
-    # TODO understand better: --foreground required otherwise we cannot
-    # kill the build with Ctrl+C if something goes wrong, can be minimized to:
-    # bash -c "eval 'timeout 5 sleep 3'"
-    common_bench_cmd "timeout --foreground 900 ../build -a '$common_arch' -o '${gem5_outdir}'" "$results_file"
-    cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt"
-    cd "${common_root_dir}/gem5/gem5"
-    git clean -xdf
-    rm -fr "${gem5_outdir}"
-  done
+  common_arch="$default_arch"
+  gem5_build_id=bench-build
+  common_gem5_build_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_out_dir)"
+  common_gem5_src_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_src_dir)"
+  results_file="${common_gem5_build_dir}/bench-build.txt"
+  git -C "${common_gem5_src_dir}" clean -xdf
+  rm -f "$results_file"
+  "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id"
+  # TODO understand better: --foreground required otherwise we cannot
+  # kill the build with Ctrl+C if something goes wrong, can be minimized to:
+  # bash -c "eval 'timeout 5 sleep 3'"
+  "${root_dir}/bench-cmd" "timeout --foreground 900 ./build-gem5 --arch '$common_arch' --gem5-build-id '$gem5_build_id'" "$results_file"
+  cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt"
+  git -C "${common_gem5_src_dir}" clean -xdf
+  "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id"
 fi
 
 if "$bench_linux_boot"; then
-  cd "${common_root_dir}"
-  ./build-all
-  ./bench-boot
-  cp "$common_bench_boot" "$new_dir"
+  cd "${root_dir}"
+  "${root_dir}/build-all"
+  "${root_dir}/bench-boot" -t 3
+  cp "$(${root_dir}/getvar bench_boot)" "$new_dir"
 fi
 
 if "$update_repo"; then

bench-all

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
+set -eu
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
 test_size=1
 OPTIND=1
 while getopts t: OPT; do
@@ -16,56 +17,83 @@ while getopts t: OPT; do
   esac
 done
 shift "$(($OPTIND - 1))"
-extra_args="$*"
-
+if [ $# -gt 1 ]; then
+  extra_args=" $*"
+else
+  extra_args=
+fi
+getvar="${root_dir}/getvar"
+common_bench_boot="$("$getvar" bench_boot)"
 caches='--caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB'
+
 bench() (
-  common_bench_cmd "./run -a ${1} ${extra_args}" "$common_bench_boot"
-  echo >> "$common_bench_boot"
+  "${root_dir}/bench-cmd" "./run --arch ${1}${extra_args}" "$common_bench_boot"
 )
+
 gem5_insts() (
-  printf "instructions $(./gem5-stat -a "$1" sim_insts)\n" >> "$common_bench_boot"
+  printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$common_bench_boot"
 )
+
 qemu_insts() (
   common_arch="$1"
-  ./qemu-trace2txt -a "$common_arch"
-  common_setup
-  printf "instructions $(wc -l "${common_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot"
+  ./qemu-trace2txt --arch "$common_arch"
+  common_qemu_trace_txt_file="$("$getvar" --arch "$common_arch" qemu_trace_txt_file)"
+  printf "instructions $(wc -l "${common_qemu_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot"
 )
+
+newline() (
+  echo >> "$common_bench_boot"
+)
+
 rm -f "${common_bench_boot}"
 
 arch=x86_64
-bench "$arch -E '/poweroff.out'"
-bench "$arch -E '/poweroff.out' -K"
+bench "${arch} --eval '/poweroff.out'"
+newline
+bench "${arch} --eval '/poweroff.out' --kvm"
+newline
 if [ "$test_size" -ge 2 ]; then
-  bench "$arch -E '/poweroff.out' -T exec_tb"
+  bench "${arch} --eval '/poweroff.out' --trace exec_tb"
   qemu_insts "$arch"
-fi
-if [ "$test_size" -ge 2 ]; then
-  bench "$arch -E 'm5 exit' -g"
+  newline
+  bench "$arch --eval 'm5 exit' --gem5"
   gem5_insts "$arch"
+  newline
 fi
-#bench "$arch -E 'm5 exit' -g -- --cpu-type=DerivO3CPU ${caches}"
+#bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=DerivO3CPU ${caches}"
 #gem5_insts "$arch"
+#newline
 
 arch=arm
-bench "$arch -E '/poweroff.out'"
+bench "$arch --eval '/poweroff.out'"
+newline
 if [ "$test_size" -ge 2 ]; then
-  bench "$arch -E '/poweroff.out' -T exec_tb"
+  bench "$arch --eval '/poweroff.out' --trace exec_tb"
   qemu_insts "$arch"
+  newline
+  #bench "$arch --eval 'm5 exit' --gem5"
+  #gem5_insts "$arch"
+  #newline
 fi
-#bench "$arch -E 'm5 exit' -g"
-#gem5_insts "$arch"
-#bench "$arch -E 'm5 exit' -g -- --cpu-type=HPI ${caches}"
-#gem5_insts "$arch"
+#if [ "$test_size" -ge 3 ]; then
+#  bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}"
+#  gem5_insts "$arch"
+#  newline
+#fi
 
 arch=aarch64
-bench "$arch -E '/poweroff.out'"
+bench "$arch --eval '/poweroff.out'"
+newline
 if [ "$test_size" -ge 2 ]; then
-  bench "$arch -E '/poweroff.out' -T exec_tb"
+  bench "$arch --eval '/poweroff.out' --trace exec_tb"
   qemu_insts "$arch"
+  newline
+  bench "$arch --eval 'm5 exit' --gem5"
+  gem5_insts "$arch"
+  newline
+fi
+if [ "$test_size" -ge 3 ]; then
+  bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}"
+  gem5_insts "$arch"
+  newline
 fi
-#bench "$arch -E 'm5 exit' -g"
-#gem5_insts "$arch"
-#bench "$arch -E 'm5 exit' -g -- --cpu-type=HPI ${caches}"
-#gem5_insts "$arch"

bench-boot

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Benchmark a command as a string and output results
+# to a file with format:
+#
+# cmd <command run>
+# time <time in seconds to finish>
+# exit_status <exit status>
+set -eu
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+# Command to benchmark
+cmd="$1"
+shift
+# Where to append write results to. Default: /dev/null.
+results_file="${1:-/dev/null}"
+printf 'cmd ' >> "$results_file"
+env time --append -f 'time %e' --output="$results_file" "${root_dir}/eeval" -a "$cmd" "$results_file"
+printf "exit_status $?\n" >> "$results_file"

bench-cmd

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import imp
+import os
+import shutil
+import sys
+
+import common
+build = imp.load_source('build', os.path.join(common.root_dir, 'build'))
+run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
+
+parser = common.get_argparse(
+    argparse_args={
+        'description': '''Bisect the Linux kernel on gem5 boots.
+
+More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#bisection
+'''},
+    default_args={
+        'gem5': True,
+        'linux_build_id': 'bisect',
+    },
+)
+args = common.setup(parser)
+# We need a clean rebuild because rebuilds at different revisions:
+# - may fail
+# - may not actually rebuild all files, e.g. on header changes
+common.rmrf(common.linux_variant_dir)
+assert build.main(args) == 0
+status = run.main(args, {
+    'eval': 'm5 exit',
+})
+if status == 125 or status == 127:
+    status = 1
+sys.exit(status)