Merge pull request #411 from OpenVicProject/benchmark/dataloading

Add Dataloading benchmarks

Author: Spartan322

.gitignore

@@ -66,6 +66,7 @@ bin/*
 *.exp
 
 tests/bin/*
+tests/benchmarks/bin/*
 
 # Build configuarion.
 /custom.py

.gitmodules

@@ -23,3 +23,6 @@
 [submodule "deps/std_function/func"]
 	path = deps/std_function/func
 	url = https://github.com/skarupke/std_function
+[submodule "tests/benchmarks/deps/nanobench"]
+	path = tests/benchmarks/deps/nanobench
+	url = https://github.com/Spartan322/nanobench

SConstruct

@@ -16,6 +16,8 @@ opts = env.SetupOptions()
 
 opts.Add(BoolVariable("build_ovsim_tests", "Build the openvic simulation unit tests", env.is_standalone))
 opts.Add(BoolVariable("run_ovsim_tests", "Run the openvic simulation unit tests", False))
+opts.Add(BoolVariable("build_ovsim_benchmarks", "Build the openvic simulation benchmarks", False))
+opts.Add(BoolVariable("run_ovsim_benchmarks", "Run the openvic simulation benchmarks", False))
 opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not env.is_standalone)))
 opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", env.is_standalone))
 
@@ -61,7 +63,10 @@ default_args = []
 if env["run_ovsim_tests"]:
     env["build_ovsim_tests"] = True
 
-if env["build_ovsim_tests"]:
+if env["run_ovsim_benchmarks"]:
+    env["build_ovsim_benchmarks"] = True
+
+if env["build_ovsim_tests"] or env["build_ovsim_benchmarks"]:
     env["build_ovsim_library"] = True
 
 if env["build_ovsim_library"]:
@@ -100,6 +105,12 @@ if env["build_ovsim_tests"]:
     if env["run_ovsim_tests"]:
         tests_env.RunUnitTest()
 
+if env["build_ovsim_benchmarks"]:
+    benchmarks_env = SConscript("tests/benchmarks/SCsub", { "env": tests_env if env["build_ovsim_tests"] else env })
+
+    if env["run_ovsim_benchmarks"]:
+        benchmarks_env.RunBenchmarks()
+
 # Add compiledb if the option is set
 if env.get("compiledb", False):
     default_args += ["compiledb"]

tests/benchmarks/SCsub

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+import os
+import subprocess
+import platform
+import sys
+
+import SCons
+from SCons.Script.SConscript import SConsEnvironment
+
+
+def BenchmarkPostAction(target=None, source=None, env=None):
+    print()
+    return subprocess.run([target[0].path]).returncode
+
+
+def BuildBenchmarks(env, **kwargs):
+    benchmark = env.Program(**kwargs)
+    env.NoCache(benchmark)
+    return benchmark
+
+
+def RunBenchmarks(env):
+    benchmark_action = env.Action(BenchmarkPostAction, None)
+    benchmark_post_action = env.AddPostAction(env.benchmark, benchmark_action)
+    env.AlwaysBuild(benchmark_post_action)
+
+
+SConsEnvironment.BuildBenchmarks = BuildBenchmarks
+SConsEnvironment.RunBenchmarks = RunBenchmarks
+
+Import("env")
+
+BINDIR = "bin"
+
+env.openvic_simulation_benchmarks = {}
+
+# For the reference:
+# - CCFLAGS are compilation flags shared between C and C++
+# - CFLAGS are for C-specific compilation flags
+# - CXXFLAGS are for C++-specific compilation flags
+# - CPPFLAGS are for pre-processor flags
+# - CPPDEFINES are for pre-processor defines
+# - LINKFLAGS are for linking flags
+
+# tweak this if you want to use different folders, or more folders, to store your source code in.
+source_path = "src"
+
+benchmarks_name = "openvic-simulation"
+benchmarks_env = env.Clone()
+benchmarks_env.Append(CPPDEFINES=["OPENVIC_SIMULATION_BENCHMARKS"])
+benchmarks_env.Append(CPPPATH=[benchmarks_env.Dir(source_path)])
+benchmarks_env.benchmarks_sources = env.GlobRecursive("*.cpp", [source_path])
+
+SConscript("deps/SCsub", {"env": benchmarks_env, "parent_env": env })
+
+benchmarks_env.benchmark = benchmarks_env.BuildBenchmarks(
+    source=benchmarks_env.benchmarks_sources,
+    target=os.path.join(BINDIR, benchmarks_name),
+    PROGSUFFIX=".benchmarks" + env["PROGSUFFIX"],
+)
+Default(benchmarks_env.benchmark)
+
+Return("benchmarks_env")

tests/benchmarks/deps/SCsub

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+import os
+import subprocess
+from pathlib import Path
+
+Import("env")
+Import("parent_env")
+
+
+def build_nanobench(env):
+    nanobench_env = env.Clone()
+
+    include_path = "nanobench/src/include"
+    source_path = "nanobench/src/test/app"
+    nanobench_env.Append(CPPPATH=[nanobench_env.Dir(include_path)])
+    sources = nanobench_env.GlobRecursive("nanobench.cpp", [source_path])
+
+    library_name = "libnanobench" + env["LIBSUFFIX"]
+    library = nanobench_env.StaticLibrary(target=os.path.join(source_path, library_name), source=sources)
+    Default(library)
+
+    env.Append(CPPPATH=[nanobench_env.Dir(include_path)])
+    env.Append(LIBPATH=[nanobench_env.Dir(source_path)])
+    env.Prepend(LIBS=[library_name])
+
+def build_snitch(env, parent_env):
+    try:
+        parent_env.snitch_env
+        SConscript("../../deps/SCsub", {"env": parent_env })
+    except(AttributeError):
+        SConscript("../../deps/SCsub", {"env": env })
+
+
+build_nanobench(env)
+build_snitch(env, parent_env)

tests/benchmarks/deps/nanobench

@@ -0,0 +1,22 @@
+#include <nanobench.h>
+
+#include "GameManager.hpp"
+#include <snitch/snitch_macros_test_case.hpp>
+
+using namespace OpenVic;
+
+TEST_CASE("Dataloading benchmark", "[benchmarks][benchmark-dataloading]") {
+	std::filesystem::path root = Dataloader::search_for_game_path();
+	Dataloader::path_vector_t roots { root };
+
+	ankerl::nanobench::Bench().epochs(10).run("Dataloading", [&] {
+		OpenVic::GameManager game_manager { []() {} };
+
+		game_manager.set_roots(roots);
+		game_manager.load_definitions(
+			[](std::string_view key, Dataloader::locale_t locale, std::string_view localisation) -> bool {
+				return true;
+			}
+		);
+	});
+}

tests/benchmarks/src/dataloading/Dataloading.cpp

@@ -108,7 +108,11 @@ def build_snitch(env):
 
     SNITCH_VERSION = "{}.{}.{}".format(major_version, minor_version, patch_version)
 
-    snitch_env = env.Clone()
+    snitch_env = None
+    try:
+        snitch_env = env.snitch_env
+    except(AttributeError):
+        snitch_env = env.Clone()
 
     snitch_env.Append(
         BUILDERS={
@@ -203,4 +207,6 @@ def build_snitch(env):
         env.Append(CPPPATH=[[snitch_env.Dir(p) for p in [source_path, include_path]]])
         Default(config)
 
+    env.snitch_env = snitch_env
+
 build_snitch(env)