Add codspeed instrumentation support for google benchmark

.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  instrumentation:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Cache build
+      uses: actions/cache@v3
+      with:
+        path: examples/google_benchmark/build-instrumentation
+        key: ${{ runner.os }}-build-instrumentation-${{ hashFiles('**/CMakeLists.txt', '**/examples/google_benchmark/**') }}
+
+    - name: Create build directory
+      run: mkdir -p examples/google_benchmark/build-instrumentation
+
+    - name: Build instrumentation benchmark example
+      run: |
+        cd examples/google_benchmark/build-instrumentation
+        cmake -DCODSPEED_MODE=instrumentation ..
+        make -j
+
+    - name: Run the benchmarks
+      uses: CodSpeedHQ/action@main
+      with:
+        run: examples/google_benchmark/build-instrumentation/benchmark_example
+        token: ${{ secrets.CODSPEED_TOKEN }}
+
+  build-walltime:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Cache build
+      uses: actions/cache@v3
+      with:
+        path: examples/google_benchmark/build-walltime
+        key: ${{ runner.os }}-build-walltime-${{ hashFiles('**/CMakeLists.txt', '**/examples/google_benchmark/**') }}
+
+    - name: Create build directory
+      run: mkdir -p examples/google_benchmark/build-walltime
+
+    - name: Build walltime benchmark example
+      run: |
+        cd examples/google_benchmark/build-walltime
+        cmake -DCODSPEED_MODE=walltime ..
+        make -j

.gitignore

@@ -1,34 +1,9 @@
-# Prerequisites
-*.d
-
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-*.smod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
 # Executables
 *.exe
 *.out
 *.app
 
-build/
+build/
+
+# Clangd cache
+.cache/

core/CMakeLists.txt

@@ -0,0 +1,30 @@
+cmake_minimum_required(VERSION 3.10)
+
+set(CODSPEED_VERSION 1.0.0)
+
+project(codspeed VERSION ${CODSPEED_VERSION} LANGUAGES CXX)
+
+# Specify the C++ standard
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+# Add the include directory
+include_directories(include)
+
+# Add the library
+add_library(codspeed src/codspeed.cpp)
+
+# Version
+add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}")
+
+# Disable valgrind compilation errors
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+    # Disable the old-style-cast warning for the specific target
+    target_compile_options(codspeed PRIVATE -Wno-old-style-cast)
+endif()
+
+# Specify the include directories for users of the library
+target_include_directories(
+    codspeed
+    PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+)

core/include/codspeed.h

@@ -0,0 +1,30 @@
+#ifndef CODSPEED_H
+#define CODSPEED_H
+
+#include <string>
+#include <vector>
+
+class CodSpeed {
+public:
+  // Public static method to access the single instance
+  static CodSpeed *getInstance() {
+    static CodSpeed instance;
+    return &instance;
+  }
+
+  // Member functions
+  void push_group(const std::string &group);
+  void pop_group();
+  void start_benchmark(const std::string &name);
+  void end_benchmark();
+
+private:
+  // Private constructor to prevent direct instantiation
+  CodSpeed();
+  std::vector<std::string> benchmarked;
+  std::string current_benchmark;
+  std::vector<std::string> group_stack;
+  bool is_instrumented;
+};
+
+#endif // CODSPEED_H

core/src/callgrind.h

@@ -0,0 +1,124 @@
+/*
+   ----------------------------------------------------------------
+
+   Notice that the following BSD-style license applies to this one
+   file (callgrind.h) only.  The rest of Valgrind is licensed under the
+   terms of the GNU General Public License, version 2, unless
+   otherwise indicated.  See the COPYING file in the source
+   distribution for details.
+
+   ----------------------------------------------------------------
+
+   This file is part of callgrind, a valgrind tool for cache simulation
+   and call tree tracing.
+
+   Copyright (C) 2003-2017 Josef Weidendorfer.  All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. The origin of this software must not be misrepresented; you must
+      not claim that you wrote the original software.  If you use this
+      software in a product, an acknowledgment in the product
+      documentation would be appreciated but is not required.
+
+   3. Altered source versions must be plainly marked as such, and must
+      not be misrepresented as being the original software.
+
+   4. The name of the author may not be used to endorse or promote
+      products derived from this software without specific prior written
+      permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+   OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+   ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+   DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+   GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   ----------------------------------------------------------------
+
+   Notice that the above BSD-style license applies to this one file
+   (callgrind.h) only.  The entire rest of Valgrind is licensed under
+   the terms of the GNU General Public License, version 2.  See the
+   COPYING file in the source distribution for details.
+
+   ----------------------------------------------------------------
+*/
+
+#ifndef __CALLGRIND_H
+#define __CALLGRIND_H
+
+#include "valgrind.h"
+
+/* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
+   This enum comprises an ABI exported by Valgrind to programs
+   which use client requests.  DO NOT CHANGE THE ORDER OF THESE
+   ENTRIES, NOR DELETE ANY -- add new ones at the end.
+
+   The identification ('C','T') for Callgrind has historical
+   reasons: it was called "Calltree" before. Besides, ('C','G') would
+   clash with cachegrind.
+ */
+
+typedef enum {
+  VG_USERREQ__DUMP_STATS = VG_USERREQ_TOOL_BASE('C', 'T'),
+  VG_USERREQ__ZERO_STATS,
+  VG_USERREQ__TOGGLE_COLLECT,
+  VG_USERREQ__DUMP_STATS_AT,
+  VG_USERREQ__START_INSTRUMENTATION,
+  VG_USERREQ__STOP_INSTRUMENTATION
+} Vg_CallgrindClientRequest;
+
+/* Dump current state of cost centers, and zero them afterwards */
+#define CALLGRIND_DUMP_STATS                                                   \
+  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DUMP_STATS, 0, 0, 0, 0, 0)
+
+/* Dump current state of cost centers, and zero them afterwards.
+   The argument is appended to a string stating the reason which triggered
+   the dump. This string is written as a description field into the
+   profile data dump. */
+#define CALLGRIND_DUMP_STATS_AT(pos_str)                                       \
+  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DUMP_STATS_AT, pos_str, 0, 0, 0, \
+                                  0)
+
+/* Zero cost centers */
+#define CALLGRIND_ZERO_STATS                                                   \
+  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__ZERO_STATS, 0, 0, 0, 0, 0)
+
+/* Toggles collection state.
+   The collection state specifies whether the happening of events
+   should be noted or if they are to be ignored. Events are noted
+   by increment of counters in a cost center */
+#define CALLGRIND_TOGGLE_COLLECT                                               \
+  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__TOGGLE_COLLECT, 0, 0, 0, 0, 0)
+
+/* Start full callgrind instrumentation if not already switched on.
+   When cache simulation is done, it will flush the simulated cache;
+   this will lead to an artificial cache warmup phase afterwards with
+   cache misses which would not have happened in reality. */
+#define CALLGRIND_START_INSTRUMENTATION                                        \
+  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__START_INSTRUMENTATION, 0, 0, 0,  \
+                                  0, 0)
+
+/* Stop full callgrind instrumentation if not already switched off.
+   This flushes Valgrinds translation cache, and does no additional
+   instrumentation afterwards, which effectivly will run at the same
+   speed as the "none" tool (ie. at minimal slowdown).
+   Use this to bypass Callgrind aggregation for uninteresting code parts.
+   To start Callgrind in this mode to ignore the setup phase, use
+   the option "--instr-atstart=no". */
+#define CALLGRIND_STOP_INSTRUMENTATION                                         \
+  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STOP_INSTRUMENTATION, 0, 0, 0,   \
+                                  0, 0)
+
+#endif /* __CALLGRIND_H */

core/src/codspeed.cpp

@@ -0,0 +1,53 @@
+#include "measurement.hpp"
+#include <codspeed.h>
+#include <iostream>
+#include <string>
+#include <vector>
+
+std::string join(const std::vector<std::string> &elements,
+                 const std::string &delimiter) {
+  std::string result;
+  for (size_t i = 0; i < elements.size(); ++i) {
+    result += elements[i];
+    if (i != elements.size() - 1) {
+      result += delimiter;
+    }
+  }
+  return result;
+}
+
+CodSpeed::CodSpeed() : is_instrumented(measurement_is_instrumented()) {
+  if (!is_instrumented) {
+    std::cerr
+        << "NOTICE: codspeed is enabled, but no performance measurement will "
+           "be made since it's running in an unknown environment."
+        << std::endl;
+  }
+  measurement_set_metadata();
+}
+
+// Member function definitions
+void CodSpeed::push_group(const std::string &group) {
+  group_stack.push_back(group);
+}
+
+void CodSpeed::pop_group() {
+  if (!group_stack.empty()) {
+    group_stack.pop_back();
+  }
+}
+
+void CodSpeed::start_benchmark(const std::string &name) {
+  current_benchmark = name;
+  measurement_start();
+}
+
+void CodSpeed::end_benchmark() {
+  measurement_stop(current_benchmark);
+  benchmarked.push_back(current_benchmark);
+  std::string action_str = is_instrumented ? "Measured" : "Checked";
+  std::string group_str =
+      group_stack.empty() ? "" : " (group: " + join(group_stack, "/") + ")";
+  std::cerr << action_str << ": " << current_benchmark << group_str
+            << std::endl;
+}

core/src/measurement.hpp

@@ -0,0 +1,32 @@
+#ifndef MEASUREMENT_H
+#define MEASUREMENT_H
+
+#include "callgrind.h"
+#include <string>
+
+inline std::string get_version() {
+#ifdef CODSPEED_VERSION
+  return {CODSPEED_VERSION};
+#else
+  return {""};
+#endif
+}
+
+inline bool measurement_is_instrumented() { return RUNNING_ON_VALGRIND; }
+
+inline void measurement_set_metadata() {
+  std::string metadata = "Metadata: codspeed-cpp " + get_version();
+  CALLGRIND_DUMP_STATS_AT(metadata.c_str());
+}
+
+inline void measurement_start() {
+  CALLGRIND_ZERO_STATS;
+  CALLGRIND_START_INSTRUMENTATION;
+}
+
+inline void measurement_stop(const std::string &name) {
+  CALLGRIND_STOP_INSTRUMENTATION;
+  CALLGRIND_DUMP_STATS_AT(name.c_str());
+};
+
+#endif // MEASUREMENT_H