From 88287533f24dcfe7d9efa202891ac043beb44cfe Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Wed, 2 Oct 2024 19:02:33 +0000 Subject: [PATCH 1/9] Add benchmark build files to git ignore --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index 1d4f942657..aede44e6cc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,19 @@ build-*/ python/build/ python/dist/ python/triton*.egg-info/ +python/*.whl python/triton/_C/*.pyd python/triton/_C/*.so python/triton/_C/*.dylib +benchmarks/dist +benchmarks/*.egg-info/ +benchmarks/**/*.so + +# Logs +inductor_log/ + # Backends copied from submodules python/triton/backends/ !python/triton/backends/__init__.py From 152a9836daec27f46868537c06367cddc0a9e357 Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Wed, 2 Oct 2024 19:06:00 +0000 Subject: [PATCH 2/9] Improve cmake for benchmarks --- benchmarks/CMakeLists.txt | 4 ++- benchmarks/cmake/FindXeTLALibrary.cmake | 4 ++- benchmarks/setup.py | 48 +++++++++++++++---------- benchmarks/xetla_kernel/CMakeLists.txt | 10 ++---- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 7d1c59ea6f..98b2543fc3 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -10,9 +10,11 @@ if(NOT WIN32) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") endif() -find_package(Python3 COMPONENTS Interpreter) +find_package(Python3 REQUIRED + COMPONENTS Development.Module) find_package(Torch REQUIRED) find_library(TORCH_PYTHON_LIBRARY torch_python PATH "${TORCH_INSTALL_PREFIX}/lib") +find_package(XeTLALibrary REQUIRED) if(USE_IPEX) string(APPEND CMAKE_CXX_FLAGS " -DUSE_IPEX") diff --git a/benchmarks/cmake/FindXeTLALibrary.cmake b/benchmarks/cmake/FindXeTLALibrary.cmake index 9d7868dae3..5599a6f882 100644 --- a/benchmarks/cmake/FindXeTLALibrary.cmake +++ b/benchmarks/cmake/FindXeTLALibrary.cmake @@ -3,13 +3,15 @@ include(FetchContent) if (NOT XeTLALibrary_FOUND) + # TODO: switch ot FetchContent_MakeAvailable once XeTLA supports it + cmake_policy(SET CMP0169 OLD) set(XeTLALibrary_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/XeTLALibrary") message(STATUS "XeTLALibrary is not specified. Will try to download XeTLA library from https://github.com/intel/xetla into ${XeTLALibrary_SOURCE_DIR}") - file(READ xetla-library.conf XeTLALibrary_TAG) + file(READ xetla_kernel/xetla-library.conf XeTLALibrary_TAG) # Strip the potential trailing newline from tag string(STRIP "${XeTLALibrary_TAG}" XeTLALibrary_TAG) FetchContent_Declare(xetla-library diff --git a/benchmarks/setup.py b/benchmarks/setup.py index 1b3f0c55cb..4b837fdf9c 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -2,7 +2,6 @@ import re import shutil import subprocess -import sysconfig import sys from setuptools import setup @@ -37,12 +36,13 @@ def run(self): self.build_extension() def build_extension(self): + # configuration + build_type = "Debug" + ninja_dir = shutil.which("ninja") # create build directories if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - # python directories - python_include_dir = sysconfig.get_path("platinclude") cmake_args = [ "-G", "Ninja", # Ninja is much faster than make @@ -50,27 +50,39 @@ def build_extension(self): ninja_dir, # Pass explicit path to ninja otherwise cmake may cache a temporary path f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}{ipex_cmake_prefix_path}", f"-DUSE_IPEX={USE_IPEX_OPTION}", - "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", - "-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=" + self.extdir, - "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + self.extdir, - "-DPython3_EXECUTABLE:FILEPATH=" + sys.executable, - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", - "-DPYTHON_INCLUDE_DIRS=" + python_include_dir, + "-DCMAKE_INSTALL_PREFIX=" + self.extdir, + "-DPython3_ROOT_DIR:FILEPATH=" + sys.exec_prefix, + "-DCMAKE_VERBOSE_MAKEFILE=TRUE", "-DCMAKE_C_COMPILER=icx", "-DCMAKE_CXX_COMPILER=icpx", + "-DCMAKE_BUILD_TYPE=" + build_type, + "-S", + self.current_dir, + "-B", + self.build_temp, ] - # configuration - build_type = "Debug" - build_args = ["--config", build_type] - cmake_args += ["-DCMAKE_BUILD_TYPE=" + build_type] max_jobs = os.getenv("MAX_JOBS", str(2 * os.cpu_count())) - build_args += ["-j" + max_jobs] + build_args = [ + "--build", + self.build_temp, + "-j" + max_jobs, + ] + + install_args = [ + "--build", + self.build_temp, + "--target", + "install", + ] env = os.environ.copy() - cmake_dir = self.build_temp - subprocess.check_call(["cmake", self.current_dir] + cmake_args, cwd=cmake_dir, env=env) - subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=cmake_dir) + print(" ".join(["cmake"] + cmake_args)) + subprocess.check_call(["cmake"] + cmake_args, env=env) + print(" ".join(["cmake"] + build_args)) + subprocess.check_call(["cmake"] + build_args) + print(" ".join(["cmake"] + install_args)) + subprocess.check_call(["cmake"] + install_args) cmake = CMakeBuild() @@ -80,4 +92,4 @@ def build_extension(self): "triton_kernels_benchmark", ], package_dir={ "triton_kernels_benchmark": "triton_kernels_benchmark", -}, package_data={"triton_kernels_benchmark": ["xetla_kernel.so"]}) +}, package_data={"triton_kernels_benchmark": ["xetla_kernel.cpython-*.so"]}) diff --git a/benchmarks/xetla_kernel/CMakeLists.txt b/benchmarks/xetla_kernel/CMakeLists.txt index beeb6dc432..439849f5c8 100644 --- a/benchmarks/xetla_kernel/CMakeLists.txt +++ b/benchmarks/xetla_kernel/CMakeLists.txt @@ -1,7 +1,3 @@ -# XeTLA library is required. -find_package(XeTLALibrary REQUIRED) -set(CMAKE_CXX_STANDARD 20) - set(XETLA_KERNEL_FLAGS ${XETLA_KERNEL_FLAGS} -fsycl -fsycl-device-code-split=per_kernel @@ -29,8 +25,7 @@ else() set(XETLA_KERNEL_FLAGS ${XETLA_KERNEL_FLAGS} "${XETLA_OFFLINE_OPTIONS}") endif() -add_library(xetla_kernel SHARED python_main.cpp) -set_target_properties(xetla_kernel PROPERTIES PREFIX "") +Python3_add_library(xetla_kernel MODULE WITH_SOABI python_main.cpp) target_compile_options(xetla_kernel PRIVATE "-fPIC") if(USE_IPEX) target_compile_options(xetla_kernel PRIVATE "-fsycl") @@ -40,7 +35,6 @@ endif() target_compile_options(xetla_kernel PUBLIC "-DXETPP_NEW_XMAIN") target_link_options(xetla_kernel PRIVATE ${XETLA_KERNEL_FLAGS}) target_link_libraries(xetla_kernel PUBLIC ${TORCH_LIBRARIES} ${TORCH_PYTHON_LIBRARY}) -target_include_directories(xetla_kernel PUBLIC "${PYTHON_INCLUDE_DIRS}") target_include_directories(xetla_kernel PUBLIC "${XeTLALibrary_INCLUDE_DIR}") if(USE_IPEX) @@ -52,3 +46,5 @@ add_subdirectory(softmax) add_subdirectory(gemm) add_subdirectory(stream_k_gemm) add_subdirectory(flash_attention) + +install(TARGETS xetla_kernel LIBRARY DESTINATION .) From 9d952c7180e4d12e038f3d67d369748faff49619 Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Wed, 2 Oct 2024 19:07:26 +0000 Subject: [PATCH 3/9] Add missing import for LIMITS in benchmarks --- benchmarks/xetla_kernel/flash_attention/fmha_forward_v5.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/xetla_kernel/flash_attention/fmha_forward_v5.h b/benchmarks/xetla_kernel/flash_attention/fmha_forward_v5.h index 7562d796dc..cd5a3de12b 100644 --- a/benchmarks/xetla_kernel/flash_attention/fmha_forward_v5.h +++ b/benchmarks/xetla_kernel/flash_attention/fmha_forward_v5.h @@ -16,6 +16,8 @@ #ifndef TRITONBENCHMARK_FMHA_FWD_V5_H #define TRITONBENCHMARK_FMHA_FWD_V5_H +#include + #include "fmha_policy_v2.h" #include "fmha_utils.h" #include "xetla.hpp" From 38c0e274b096ce783ed625de1945ae05e376366c Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Thu, 3 Oct 2024 16:52:42 +0000 Subject: [PATCH 4/9] Trigger benchamrks' cmake build only when needed and use build_type --- benchmarks/setup.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/benchmarks/setup.py b/benchmarks/setup.py index 4b837fdf9c..b7a3205bbd 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -5,6 +5,7 @@ import sys from setuptools import setup +from setuptools.command.build_ext import build_ext as build_ext_orig import torch @@ -17,10 +18,11 @@ class CMakeBuild(): - def __init__(self): + def __init__(self, build_type="Debug"): self.current_dir = os.path.abspath(os.path.dirname(__file__)) self.build_temp = self.current_dir + "/build/temp" self.extdir = self.current_dir + "/triton_kernels_benchmark" + self.build_type = build_type def run(self): try: @@ -36,9 +38,6 @@ def run(self): self.build_extension() def build_extension(self): - # configuration - build_type = "Debug" - ninja_dir = shutil.which("ninja") # create build directories if not os.path.exists(self.build_temp): @@ -55,7 +54,7 @@ def build_extension(self): "-DCMAKE_VERBOSE_MAKEFILE=TRUE", "-DCMAKE_C_COMPILER=icx", "-DCMAKE_CXX_COMPILER=icpx", - "-DCMAKE_BUILD_TYPE=" + build_type, + "-DCMAKE_BUILD_TYPE=" + self.build_type, "-S", self.current_dir, "-B", @@ -85,11 +84,23 @@ def build_extension(self): subprocess.check_call(["cmake"] + install_args) -cmake = CMakeBuild() -cmake.run() +class build_ext(build_ext_orig): + + def run(self): + self.build_cmake() + super().run() + + def build_cmake(self): + DEBUG_OPTION = os.getenv("DEBUG", "0") + build_type = "Debug" if self.debug or DEBUG_OPTION == "1" else "Release" + cmake = CMakeBuild(build_type) + cmake.run() + setup(name="triton-kernels-benchmark", packages=[ "triton_kernels_benchmark", ], package_dir={ "triton_kernels_benchmark": "triton_kernels_benchmark", -}, package_data={"triton_kernels_benchmark": ["xetla_kernel.cpython-*.so"]}) +}, package_data={"triton_kernels_benchmark": ["xetla_kernel.cpython-*.so"]}, cmdclass={ + "build_ext": build_ext, +}) From b5f28b403e1728df3c24f4726ef5c24301322270 Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Thu, 3 Oct 2024 17:23:33 +0000 Subject: [PATCH 5/9] Remove redundant cmake version check at benchmarks --- benchmarks/setup.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/benchmarks/setup.py b/benchmarks/setup.py index b7a3205bbd..ff7f824be9 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -1,5 +1,4 @@ import os -import re import shutil import subprocess import sys @@ -25,16 +24,6 @@ def __init__(self, build_type="Debug"): self.build_type = build_type def run(self): - try: - out = subprocess.check_output(["cmake", "--version"]) - except OSError as error: - raise RuntimeError("CMake must be installed") from error - - match = re.search(r"version\s*(?P\d+)\.(?P\d+)([\d.]+)?", out.decode()) - cmake_major, cmake_minor = int(match.group("major")), int(match.group("minor")) - if (cmake_major, cmake_minor) < (3, 18): - raise RuntimeError("CMake >= 3.18.0 is required") - self.build_extension() def build_extension(self): From f8645f4951bc81df6f3817669edefee75fe6f35c Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Thu, 3 Oct 2024 17:32:58 +0000 Subject: [PATCH 6/9] Incapsulate IPEX check in benchamrks --- benchmarks/setup.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/benchmarks/setup.py b/benchmarks/setup.py index ff7f824be9..d70f43de98 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -8,12 +8,6 @@ import torch -ipex_cmake_prefix_path = "" -USE_IPEX_OPTION = os.getenv("USE_IPEX", "1") -if USE_IPEX_OPTION == "1": - import intel_extension_for_pytorch - ipex_cmake_prefix_path = f";{intel_extension_for_pytorch.cmake_prefix_path}" - class CMakeBuild(): @@ -22,10 +16,20 @@ def __init__(self, build_type="Debug"): self.build_temp = self.current_dir + "/build/temp" self.extdir = self.current_dir + "/triton_kernels_benchmark" self.build_type = build_type + self.cmake_prefix_paths = [torch.utils.cmake_prefix_path] + self.use_ipex = False def run(self): + self.check_ipex() self.build_extension() + def check_ipex(self): + self.use_ipex = os.getenv("USE_IPEX", "1") == "1" + if not self.use_ipex: + return + import intel_extension_for_pytorch + self.cmake_prefix_paths.append(intel_extension_for_pytorch.cmake_prefix_path) + def build_extension(self): ninja_dir = shutil.which("ninja") # create build directories @@ -36,8 +40,8 @@ def build_extension(self): "Ninja", # Ninja is much faster than make "-DCMAKE_MAKE_PROGRAM=" + ninja_dir, # Pass explicit path to ninja otherwise cmake may cache a temporary path - f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}{ipex_cmake_prefix_path}", - f"-DUSE_IPEX={USE_IPEX_OPTION}", + "-DCMAKE_PREFIX_PATH=" + ";".join(self.cmake_prefix_paths), + "-DUSE_IPEX=" + ("1" if self.use_ipex else "0"), "-DCMAKE_INSTALL_PREFIX=" + self.extdir, "-DPython3_ROOT_DIR:FILEPATH=" + sys.exec_prefix, "-DCMAKE_VERBOSE_MAKEFILE=TRUE", From 7068f3606098a0eac759f70ed707ba16e9f94899 Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Tue, 8 Oct 2024 20:01:00 +0000 Subject: [PATCH 7/9] Integrate cmake into cmdclass for benchmarks --- benchmarks/setup.py | 54 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/benchmarks/setup.py b/benchmarks/setup.py index d70f43de98..4eedd6774f 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -3,8 +3,12 @@ import subprocess import sys +from distutils import log +from distutils.dir_util import remove_tree +from distutils.command.clean import clean as _clean +from distutils.command.build import build as _build + from setuptools import setup -from setuptools.command.build_ext import build_ext as build_ext_orig import torch @@ -27,9 +31,18 @@ def check_ipex(self): self.use_ipex = os.getenv("USE_IPEX", "1") == "1" if not self.use_ipex: return - import intel_extension_for_pytorch + try: + import intel_extension_for_pytorch + except ImportError: + log.warn("ipex is not installed trying to build without ipex") + self.use_ipex = False + return self.cmake_prefix_paths.append(intel_extension_for_pytorch.cmake_prefix_path) + def check_call(self, *popenargs, **kwargs): + print(" ".join(popenargs[0])) + subprocess.check_call(*popenargs, **kwargs) + def build_extension(self): ninja_dir = shutil.which("ninja") # create build directories @@ -69,15 +82,19 @@ def build_extension(self): ] env = os.environ.copy() - print(" ".join(["cmake"] + cmake_args)) - subprocess.check_call(["cmake"] + cmake_args, env=env) - print(" ".join(["cmake"] + build_args)) - subprocess.check_call(["cmake"] + build_args) - print(" ".join(["cmake"] + install_args)) - subprocess.check_call(["cmake"] + install_args) + self.check_call(["cmake"] + cmake_args, env=env) + self.check_call(["cmake"] + build_args) + self.check_call(["cmake"] + install_args) + def clean(self): + if os.path.exists(self.build_temp): + remove_tree(self.build_temp, dry_run=self.dry_run) + else: + log.warn("'%s' does not exist -- can't clean it", os.path.relpath(self.build_temp, + os.path.dirname(__file__))) -class build_ext(build_ext_orig): + +class build(_build): def run(self): self.build_cmake() @@ -85,15 +102,30 @@ def run(self): def build_cmake(self): DEBUG_OPTION = os.getenv("DEBUG", "0") - build_type = "Debug" if self.debug or DEBUG_OPTION == "1" else "Release" + debug = DEBUG_OPTION == "1" + if hasattr(self, "debug"): + debug = debug or self.debug + build_type = "Debug" if debug else "Release" cmake = CMakeBuild(build_type) cmake.run() +class clean(_clean): + + def run(self): + self.clean_cmake() + super().run() + + def clean_cmake(self): + cmake = CMakeBuild() + cmake.clean() + + setup(name="triton-kernels-benchmark", packages=[ "triton_kernels_benchmark", ], package_dir={ "triton_kernels_benchmark": "triton_kernels_benchmark", }, package_data={"triton_kernels_benchmark": ["xetla_kernel.cpython-*.so"]}, cmdclass={ - "build_ext": build_ext, + "build": build, + "clean": clean, }) From f4953ee86d785e4037cc76ecaeabd89f1331de46 Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Thu, 10 Oct 2024 14:00:43 +0000 Subject: [PATCH 8/9] Add ext_modules for install build --- benchmarks/setup.py | 56 ++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/benchmarks/setup.py b/benchmarks/setup.py index 4eedd6774f..48313a15a3 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -3,25 +3,39 @@ import subprocess import sys -from distutils import log -from distutils.dir_util import remove_tree -from distutils.command.clean import clean as _clean -from distutils.command.build import build as _build +# TODO: update once there is replacement for clean: +# https://github.com/pypa/setuptools/discussions/2838 +from distutils import log # pylint: disable=[deprecated-module] +from distutils.dir_util import remove_tree # pylint: disable=[deprecated-module] +from distutils.command.clean import clean as _clean # pylint: disable=[deprecated-module] -from setuptools import setup +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext import torch +class CMakeExtension(Extension): + + def __init__(self, name): + # don't invoke the original build_ext for this special extension + super().__init__(name, sources=[]) + + class CMakeBuild(): - def __init__(self, build_type="Debug"): + def __init__(self, debug=False, dry_run=False): self.current_dir = os.path.abspath(os.path.dirname(__file__)) self.build_temp = self.current_dir + "/build/temp" self.extdir = self.current_dir + "/triton_kernels_benchmark" - self.build_type = build_type + self.build_type = self.get_build_type(debug) self.cmake_prefix_paths = [torch.utils.cmake_prefix_path] self.use_ipex = False + self.dry_run = dry_run + + def get_build_type(self, debug): + DEBUG_OPTION = os.getenv("DEBUG", "0") + return "Debug" if debug or (DEBUG_OPTION == "1") else "Release" def run(self): self.check_ipex() @@ -41,7 +55,8 @@ def check_ipex(self): def check_call(self, *popenargs, **kwargs): print(" ".join(popenargs[0])) - subprocess.check_call(*popenargs, **kwargs) + if not self.dry_run: + subprocess.check_call(*popenargs, **kwargs) def build_extension(self): ninja_dir = shutil.which("ninja") @@ -94,31 +109,20 @@ def clean(self): os.path.dirname(__file__))) -class build(_build): +class build_ext(_build_ext): def run(self): - self.build_cmake() - super().run() - - def build_cmake(self): - DEBUG_OPTION = os.getenv("DEBUG", "0") - debug = DEBUG_OPTION == "1" - if hasattr(self, "debug"): - debug = debug or self.debug - build_type = "Debug" if debug else "Release" - cmake = CMakeBuild(build_type) + cmake = CMakeBuild(debug=self.debug, dry_run=self.dry_run) cmake.run() + super().run() class clean(_clean): def run(self): - self.clean_cmake() - super().run() - - def clean_cmake(self): - cmake = CMakeBuild() + cmake = CMakeBuild(dry_run=self.dry_run) cmake.clean() + super().run() setup(name="triton-kernels-benchmark", packages=[ @@ -126,6 +130,6 @@ def clean_cmake(self): ], package_dir={ "triton_kernels_benchmark": "triton_kernels_benchmark", }, package_data={"triton_kernels_benchmark": ["xetla_kernel.cpython-*.so"]}, cmdclass={ - "build": build, + "build_ext": build_ext, "clean": clean, -}) +}, ext_modules=[CMakeExtension("triton_kernels_benchmark")]) From 368ef37b16e8df7ed9e7f43695b003032ccd6576 Mon Sep 17 00:00:00 2001 From: Yevhenii Havrylko Date: Thu, 10 Oct 2024 23:03:22 +0000 Subject: [PATCH 9/9] Apply discussion fixes --- benchmarks/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/setup.py b/benchmarks/setup.py index 48313a15a3..7497f02585 100644 --- a/benchmarks/setup.py +++ b/benchmarks/setup.py @@ -54,7 +54,7 @@ def check_ipex(self): self.cmake_prefix_paths.append(intel_extension_for_pytorch.cmake_prefix_path) def check_call(self, *popenargs, **kwargs): - print(" ".join(popenargs[0])) + log.info(" ".join(popenargs[0])) if not self.dry_run: subprocess.check_call(*popenargs, **kwargs) @@ -69,7 +69,7 @@ def build_extension(self): "-DCMAKE_MAKE_PROGRAM=" + ninja_dir, # Pass explicit path to ninja otherwise cmake may cache a temporary path "-DCMAKE_PREFIX_PATH=" + ";".join(self.cmake_prefix_paths), - "-DUSE_IPEX=" + ("1" if self.use_ipex else "0"), + f"-DUSE_IPEX={int(self.use_ipex)}", "-DCMAKE_INSTALL_PREFIX=" + self.extdir, "-DPython3_ROOT_DIR:FILEPATH=" + sys.exec_prefix, "-DCMAKE_VERBOSE_MAKEFILE=TRUE",