[Benchmark] Compare clang-tidy SCU vs. classic

.clang-tidy

@@ -0,0 +1,22 @@
+---
+
+# For now, we just use the default checks with some additions and exceptions
+# based on what actually passes. In the future, this should be turned into a
+# blacklist, like so:
+#
+#   ###
+#   # First enable all, then disable the ones we do not want:
+#   #  * -abseil-*: not applicable
+#   #  * -android-*: not applicable
+#   #  * -objc-*: not applicable
+#   #  * -fuchsia-*: not applicable
+#   #  * -mpi-*: not applicable
+#   #  * -performance-*: not relevant
+#   #  * -llvm-include-order': not our style
+#   # Checks: '*,-abseil-*,-android-*,-objc-*,-fuchsia-*,-mpi-*,-performance-*,-llvm-include-order'
+#   ###
+#
+Checks: 'readability-braces-around-statements,google-readability-braces-around-statements,hicpp-braces-around-statements,readability-container-size-empty,readability-inconsistent-declaration-parameter-name,bugprone-macro-parentheses,bugprone-bool-pointer-implicit-conversion,bugprone-suspicious-string-compare,cppcoreguidelines-pro-type-member-init'
+
+# Uncomment this to make clang-tidy fail on any findings:
+WarningsAsErrors: '*'

.travis.yml

@@ -12,39 +12,7 @@ branches:
 
 jobs:
     include:
-        - name: "Build with GCC, run tests"
-          env: TRAVIS_CACHE_ID=focal
-          cache:
-              directories:
-                  - $HOME/.ccache-focal-gcc
-                  - $HOME/.tox-cache
-          script:
-              # Fetch our current Ubuntu 20.04 CI image from Docker Hub
-              # (automatically built by Docker):
-              - docker pull hlwm/ci:focal
-
-              # Build ad-hoc image, using hub image as cache
-              # (=> does not actually build from scratch if the Dockerfile is
-              # the same):
-              - docker build -t focal --cache-from hlwm/ci:focal - <ci/Dockerfile.ci-focal
-
-              # Perform actual tasks within temporary Docker image
-              - docker run -u $UID --privileged -t --rm
-                --volume=/etc/passwd:/etc/passwd:ro
-                --volume=$HOME/.ccache-focal-gcc:/.ccache:rw
-                --volume=$PWD:/hlwm:rw
-                --volume=$HOME/.tox-cache:/hlwm/.tox:rw
-                focal sh -c '
-                    /hlwm/ci/build.py --build-dir=/hlwm/build --cxx=g++-9 --cc=gcc-9 --build-type=Debug --ccache=/.ccache --compile --run-tests
-                '
-              - bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info
-
         - name: "Build with Clang, run linters and static analyzers"
-          env: TRAVIS_CACHE_ID=focal-linter
-          cache:
-              directories:
-                  - $HOME/.ccache-eoan-clang
-                  - $HOME/.tox-cache
           script:
               # Fetch our current Ubuntu 20.04 CI image from Docker Hub
               # (automatically built by Docker):
@@ -59,41 +27,14 @@ jobs:
               - docker run -u $UID --privileged -t --rm
                 --volume=/etc/passwd:/etc/passwd:ro
                 --volume=$PWD:/hlwm:rw
-                --volume=$HOME/.ccache-eoan-clang:/.ccache:rw
-                --volume=$HOME/.tox-cache:/hlwm/.tox:rw
-                focal sh -c '
-                    /hlwm/ci/build.py --build-dir=/hlwm/build --cxx=/hlwm/ci/clang++-and-tidy.sh --cc=/hlwm/ci/clang-and-tidy.sh --build-type=Debug --ccache=/.ccache --compile --check-using-std --iwyu --flake8
+                focal bash -c '
+                    set -o errexit;
+                    for i in {1..5}; do
+                      time /hlwm/ci/build.py --build-dir=/hlwm/build --cxx=clang++-10 --cc=clang-10 --build-type=Debug --clang-tidy;
+                      rm -rf /hlwm/build;
+                    done;
+                    for i in {1..5}; do
+                      time /hlwm/ci/build.py --build-dir=/hlwm/build --cxx=clang++-10 --cc=clang-10 --build-type=Debug --unity-build --clang-tidy;
+                      rm -rf /hlwm/build;
+                    done
                 '
-
-        - name: "Build for 32bit with ancient GCC on Ubuntu 14.04"
-          env: TRAVIS_CACHE_ID=trusty
-          cache:
-              directories:
-                  - $HOME/.ccache-gcc-4.8
-          script:
-              # Fetch our current Ubuntu 14.04 CI image from Docker Hub
-              # (automatically built by Docker):
-              - docker pull hlwm/ci:trusty
-
-              # Build ad-hoc image, using hub image as cache
-              # (=> does not actually build from scratch if the Dockerfile is
-              # the same):
-              - docker build -t trusty --cache-from hlwm/ci:trusty -
-                <ci/Dockerfile.ci-trusty
-
-              # Build with gcc-4.8 using ad-hoc image (plain bash one-liner
-              # because Python is too old):
-              - docker run -u $UID -t --rm
-                --volume=$PWD:/hlwm:rw
-                --volume=$HOME/.ccache-gcc-4.8:/.ccache:rw
-                --env CC=gcc-4.8
-                --env CXX=g++-4.8
-                --env CXXFLAGS=-m32
-                --env CFLAGS=-m32
-                trusty bash -c '
-                    cd /tmp &&
-                    ccache -z --max-size=500M &&
-                    cmake -GNinja -DWITH_DOCUMENTATION=NO -DENABLE_CCACHE=YES /hlwm &&
-                    time ninja -v -k10 &&
-                    ccache -s
-                    '

ci/build.py

@@ -5,28 +5,49 @@
 import re
 import subprocess as sp
 import sys
+import time
+import datetime
 from pathlib import Path
 
 
+def check_call_timed(*args, **kwargs):
+    """
+    Wrapper around subprocess.check_call, but with time measurement
+    """
+    start_time = time.perf_counter()
+    ret = sp.check_call(*args, **kwargs)
+    end_time = time.perf_counter()
+    duration = datetime.timedelta(seconds=end_time - start_time)
+
+    label = args[0]
+    if not isinstance(label, str):
+        label = label[0]
+    print(f'Duration for {label}: {duration}')
+
+    return ret
+
+
 def tox(tox_args, build_dir):
     """
     Prepare environment for tox and execute it with the given arguments
     """
     tox_env = os.environ.copy()
     tox_env['PWD'] = build_dir
-    sp.check_call(f'tox {tox_args}', shell=True, cwd=build_dir, env=tox_env)
+    check_call_timed(f'tox {tox_args}', shell=True, cwd=build_dir, env=tox_env)
 
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--build-dir', type=str, required=True)
 parser.add_argument('--build-type', type=str, choices=('Release', 'Debug'), required=True)
 parser.add_argument('--compile', action='store_true')
+parser.add_argument('--unity-build', action='store_true')
 parser.add_argument('--run-tests', action='store_true')
 parser.add_argument('--build-docs', action='store_true')
 parser.add_argument('--cxx', type=str)
 parser.add_argument('--cc', type=str)
 parser.add_argument('--check-using-std', action='store_true')
 parser.add_argument('--iwyu', action='store_true')
+parser.add_argument('--clang-tidy', action='store_true')
 parser.add_argument('--flake8', action='store_true')
 parser.add_argument('--ccache', nargs='?', metavar='ccache dir', type=str,
                     const=os.environ.get('CCACHE_DIR') or True)
@@ -37,7 +58,7 @@ def tox(tox_args, build_dir):
 build_dir.mkdir(exist_ok=True)
 
 if args.check_using_std:
-    sp.check_call(['./ci/check-using-std.sh'], cwd=repo)
+    check_call(['./ci/check-using-std.sh'], cwd=repo)
 
 if args.ccache:
     if args.ccache is not True:
@@ -63,15 +84,6 @@ def tox(tox_args, build_dir):
     'CXX': args.cxx,
     'CFLAGS': '--coverage -Werror -fsanitize=address,leak,undefined',
     'CXXFLAGS': '--coverage -Werror -fsanitize=address,leak,undefined',
-
-    # Hash-verifying the compiler is required when building with
-    # clang-and-tidy.sh (because the script's mtime is not stable) and for
-    # other cases, the overhead is minimal):
-    'CCACHE_COMPILERCHECK': 'content',
-
-    # In case clang-and-tidy.sh is used for building, it will need this to call
-    # clang-tidy:
-    'CLANG_TIDY_BUILD_DIR': str(build_dir),
 })
 
 cmake_args = [
@@ -82,10 +94,17 @@ def tox(tox_args, build_dir):
     f'-DENABLE_CCACHE={"YES" if args.ccache else "NO"}',
 ]
 
-sp.check_call(['cmake', *cmake_args, repo], cwd=build_dir, env=build_env)
+if args.unity_build:
+    cmake_args += [
+    '-DCMAKE_UNITY_BUILD=ON',
+    '-DCMAKE_UNITY_BUILD_BATCH_SIZE=0',
+    ]
+
+check_call_timed(['cmake', *cmake_args, repo], cwd=build_dir, env=build_env)
+sp.run('echo $(grep \'"command"\' compile_commands.json|wc -l) compile commands generated', shell=True, cwd=build_dir)
 
 if args.compile:
-    sp.check_call(['bash', '-c', 'time ninja -v -k 10'], cwd=build_dir, env=build_env)
+    check_call_timed(['bash', '-c', 'ninja -v -k 10'], cwd=build_dir, env=build_env)
 
 if args.ccache:
     sp.check_call(['ccache', '-s'])
@@ -120,6 +139,13 @@ def tox(tox_args, build_dir):
         print("")
         sys.exit(1)
 
+if args.clang_tidy:
+    check_call_timed(f'python3 /usr/lib/llvm-10/share/clang/run-clang-tidy.py -extra-arg=-Wno-unknown-warning-option -header-filter=^{repo}/.* {repo}',
+                  shell=True,
+                  cwd=build_dir,
+                  stderr=sp.PIPE,
+                  )
+
 if args.flake8:
     tox('-e flake8', build_dir)