diff --git a/CHANGELOG.md b/CHANGELOG.md index aa531908591f..606b2d21dff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Also, that release drops support for Python 3.9, making Python 3.10 the minimum * Unified public API definitions in `dpnp.linalg` and `dpnp.scipy` submodules [#2663](https://github.com/IntelPython/dpnp/pull/2663) * Aligned the signature of `dpnp.reshape` function with Python array API by making `shape` a required argument [#2673](https://github.com/IntelPython/dpnp/pull/2673) * Unified `dpnp` public API exports by consolidating function exports in `__init__.py` and removing wildcard imports [#2665](https://github.com/IntelPython/dpnp/pull/2665) [#2666](https://github.com/IntelPython/dpnp/pull/2666) +* Changed the build scripts and documentation due to `python setup.py develop` deprecation notice [#2172](https://github.com/IntelPython/dpnp/pull/2172) ### Deprecated diff --git a/CMakeLists.txt b/CMakeLists.txt index 66f5c776b52d..43ce3aab1b09 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,7 +107,7 @@ set(DPNP_TARGET_CUDA Set to a truthy value (e.g., ON, TRUE) to use default architecture (sm_50), \ or to a specific architecture like sm_80." ) -set(HIP_TARGETS "" CACHE STRING "HIP architecture for target") +set(DPNP_TARGET_HIP "" CACHE STRING "HIP architecture for target") set(_dpnp_sycl_targets) set(_use_onemath OFF) @@ -135,18 +135,22 @@ if("x${DPNP_SYCL_TARGETS}" STREQUAL "x") set(_use_onemath_cuda ON) endif() - if(HIP_TARGETS) - if(HIP_TARGETS MATCHES "^gfx") + if(DPNP_TARGET_HIP) + if(DPNP_TARGET_HIP MATCHES "^gfx") if("x${_dpnp_sycl_targets}" STREQUAL "x") - set(_dpnp_sycl_targets "amd_gpu_${HIP_TARGETS},spir64-unknown-unknown") + set(_dpnp_sycl_targets + "amd_gpu_${DPNP_TARGET_HIP},spir64-unknown-unknown" + ) else() - set(_dpnp_sycl_targets "amd_gpu_${HIP_TARGETS},${_dpnp_sycl_targets}") + set(_dpnp_sycl_targets + "amd_gpu_${DPNP_TARGET_HIP},${_dpnp_sycl_targets}" + ) endif() set(_use_onemath_hip ON) else() message( FATAL_ERROR - "Invalid value for HIP_TARGETS: \"${HIP_TARGETS}\". " + "Invalid value for DPNP_TARGET_HIP: \"${DPNP_TARGET_HIP}\". " "Expected an architecture name starting with 'gfx', e.g. 'gfx1030'." ) endif() @@ -161,8 +165,11 @@ else() if("${DPNP_SYCL_TARGETS}" MATCHES "amd_gpu_") set(_use_onemath_hip ON) - if("x${HIP_TARGETS}" STREQUAL "x") - message(FATAL_ERROR "HIP_TARGETS must be specified when using HIP backend") + if("x${DPNP_TARGET_HIP}" STREQUAL "x") + message( + FATAL_ERROR + "DPNP_TARGET_HIP must be specified when using HIP backend" + ) endif() endif() diff --git a/doc/0.builddoc.sh b/doc/0.builddoc.sh deleted file mode 100755 index f10b4a5cc22d..000000000000 --- a/doc/0.builddoc.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -BUILDDOCDIR=$(dirname "$(readlink -e "${BASH_SOURCE[0]}")") -ROOTDIR=$BUILDDOCDIR/.. - -cd "$ROOTDIR" || exit 1 -python setup.py develop - -cd "$BUILDDOCDIR" || exit 2 -make clean -make html diff --git a/doc/quick_start_guide.rst b/doc/quick_start_guide.rst index 92c506c0fd81..6226a655c333 100644 --- a/doc/quick_start_guide.rst +++ b/doc/quick_start_guide.rst @@ -112,13 +112,15 @@ To build and install the package on Linux OS, run: .. code-block:: bash - python setup.py install -- -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx + python setup.py build_ext --inplace -- -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx + python -m pip install -e . To build and install the package on Windows OS, run: .. code-block:: bash - python setup.py install -- -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icx + python setup.py build_ext --inplace -- -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icx + python -m pip install -e . Alternatively, to develop on Linux OS, you can use the driver script: diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py new file mode 100644 index 000000000000..a33ed269e1ed --- /dev/null +++ b/scripts/_build_helper.py @@ -0,0 +1,243 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +import os +import shutil +import subprocess +import sys +import warnings + + +def get_dpctl_cmake_dir(): + """ + If dpctl is locally built using `script/build_locally.py`, it is needed + to pass the -DDpctl_ROOT=$(python -m dpctl --cmakedir) during the build. + If dpctl is conda installed, it is optional to pass this parameter. + + """ + + process = subprocess.Popen( + ["python", "-m", "dpctl", "--cmakedir"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + output, error = process.communicate() + if process.returncode == 0: + return output.decode("utf-8").strip() + + raise RuntimeError( + "Failed to retrieve dpctl cmake directory: " + + error.decode("utf-8").strip() + ) + + +def resolve_compilers( + oneapi: bool, + c_compiler: str, + cxx_compiler: str, + compiler_root: str, +): + is_linux = "linux" in sys.platform + + if oneapi or ( + c_compiler is None and cxx_compiler is None and compiler_root is None + ): + return "icx", ("icpx" if is_linux else "icx") + + if ( + (c_compiler is None or not os.path.isabs(c_compiler)) + and (cxx_compiler is None or not os.path.isabs(cxx_compiler)) + and (not compiler_root or not os.path.exists(compiler_root)) + ): + raise RuntimeError( + "--compiler-root option must be set when using non-default DPC++ " + "layout unless absolute paths are provided for both compilers" + ) + + # default values + if c_compiler is None: + c_compiler = "icx" + if cxx_compiler is None: + cxx_compiler = "icpx" if is_linux else "icx" + + for name, opt_name in ( + (c_compiler, "--c-compiler"), + (cxx_compiler, "--cxx-compiler"), + ): + if os.path.isabs(name): + path = name + else: + path = os.path.join(compiler_root, name) + if not os.path.exists(path): + raise RuntimeError(f"{opt_name} value {name} not found") + return c_compiler, cxx_compiler + + +def resolve_onemath( + onemath: bool, + onemath_dir: str, + target_cuda: str = None, + target_hip: str = None, + onemkl_interfaces: bool = False, + onemkl_interfaces_dir: str = None, +): + # always enable build with oneMath i/f when oneMath path is passed + if onemath_dir: + onemath = True + + # always enable build with oneMath i/f for CUDA or HIP target + if target_cuda or target_hip: + onemath = True + + # TODO: onemkl_interfaces and onemkl_interfaces_dir are deprecated in + # dpnp-0.19.0 and should be removed in dpnp-0.20.0. + if onemkl_interfaces: + warnings.warn( + "Using 'onemkl_interfaces' is deprecated. Please use 'onemath' instead.", + DeprecationWarning, + stacklevel=1, + ) + onemath = True + if onemkl_interfaces_dir is not None: + warnings.warn( + "Using 'onemkl_interfaces_dir' is deprecated. Please use 'onemath_dir' instead.", + DeprecationWarning, + stacklevel=1, + ) + onemath_dir = onemkl_interfaces_dir + return onemath, onemath_dir + + +def run(cmd: list[str], env: dict[str, str] = None, cwd: str = None): + print("+", " ".join(cmd)) + subprocess.check_call( + cmd, env=env or os.environ.copy(), cwd=cwd or os.getcwd() + ) + + +def capture_cmd_output(cmd: list[str], cwd: str = None): + print("+", " ".join(cmd)) + return ( + subprocess.check_output(cmd, cwd=cwd or os.getcwd()) + .decode("utf-8") + .strip("\n") + ) + + +def err(msg: str, script: str): + raise RuntimeError(f"[{script}] error: {msg}") + + +def log_cmake_args(cmake_args: list[str], script: str): + print(f"[{script}] Using CMake args:\n{' '.join(cmake_args)}") + + +def make_cmake_args( + c_compiler: str = None, + cxx_compiler: str = None, + dpctl_cmake_dir: str = None, + onemath: bool = False, + onemath_dir: str = None, + verbose: bool = False, + other_opts: str = None, +): + args = [ + f"-DCMAKE_C_COMPILER:PATH={c_compiler}" if c_compiler else "", + f"-DCMAKE_CXX_COMPILER:PATH={cxx_compiler}" if cxx_compiler else "", + f"-DDpctl_ROOT={dpctl_cmake_dir}" if dpctl_cmake_dir else "", + ] + + if onemath: + args.append("-DDPNP_USE_ONEMATH=ON") + if onemath_dir: + args.append(f"-DDPNP_ONEMATH_DIR={onemath_dir}") + + if verbose: + args.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") + if other_opts: + args.extend(other_opts.split()) + + return args + + +def build_extension( + setup_dir: str, + env: dict[str, str], + cmake_args: list[str], + cmake_executable: str = None, + generator: str = None, + build_type: str = None, +): + cmd = [sys.executable, "setup.py", "build_ext", "--inplace"] + if cmake_executable: + cmd.append(f"--cmake-executable={cmake_executable}") + if generator: + cmd.append(f"--generator={generator}") + if build_type: + cmd.append(f"--build-type={build_type}") + if cmake_args: + cmd.append("--") + cmd += cmake_args + run( + cmd, + env=env, + cwd=setup_dir, + ) + + +def install_editable(setup_dir: str, env: dict[str, str]): + run( + [ + sys.executable, + "-m", + "pip", + "install", + "-e", + ".", + "--no-build-isolation", + ], + env=env, + cwd=setup_dir, + ) + + +def clean_build_dir(setup_dir: str): + if ( + not isinstance(setup_dir, str) + or not setup_dir + or not os.path.isdir(setup_dir) + ): + raise RuntimeError(f"Invalid setup directory provided: '{setup_dir}'") + target = os.path.join(setup_dir, "_skbuild") + if os.path.exists(target): + print(f"Cleaning build directory: {target}") + try: + shutil.rmtree(target) + except Exception as e: + print(f"Failed to remove build directory: '{target}'") + raise e diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 1197de9d9455..e17fc8e478fe 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -26,290 +26,211 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** +import argparse import os -import subprocess import sys -import warnings -warnings.simplefilter("default", DeprecationWarning) - - -def run( - use_oneapi=True, - build_type="Release", - c_compiler=None, - cxx_compiler=None, - compiler_root=None, - cmake_executable=None, - verbose=False, - cmake_opts="", - target_cuda=None, - target_hip=None, - onemkl_interfaces=False, - onemkl_interfaces_dir=None, - onemath=False, - onemath_dir=None, -): - build_system = None - - if "linux" in sys.platform: - build_system = "Ninja" - elif sys.platform in ["win32", "cygwin"]: - build_system = "Ninja" - else: - raise AssertionError(sys.platform + " not supported") - - setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - cmake_args = [ - sys.executable, - "setup.py", - "develop", - ] - if cmake_executable: - cmake_args += [ - "--cmake-executable=" + cmake_executable, - ] - - # if dpctl is locally built using `script/build_locally.py`, it is needed - # to pass the -DDpctl_ROOT=$(python -m dpctl --cmakedir) - # if dpctl is conda installed, it is optional to pass this parameter - process = subprocess.Popen( - ["python", "-m", "dpctl", "--cmakedir"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - output, error = process.communicate() - if process.returncode == 0: - cmake_dir = output.decode("utf-8").strip() - else: - raise RuntimeError( - "Failed to retrieve dpctl cmake directory: " - + error.decode("utf-8").strip() - ) - - cmake_args += [ - "--build-type=" + build_type, - "--generator=" + build_system, - "--", - "-DCMAKE_C_COMPILER:PATH=" + c_compiler, - "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, - "-DDpctl_ROOT=" + cmake_dir, - ] - if verbose: - cmake_args += [ - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", - ] - if cmake_opts: - cmake_args += cmake_opts.split() - if use_oneapi: - if "DPL_ROOT" in os.environ: - os.environ["DPL_ROOT_HINT"] = os.environ["DPL_ROOT"] - - # TODO: onemkl_interfaces and onemkl_interfaces_dir are deprecated in - # dpnp-0.19.0 and should be removed in dpnp-0.20.0. - if onemkl_interfaces: - warnings.warn( - "Using 'onemkl_interfaces' is deprecated. Please use 'onemath' instead.", - DeprecationWarning, - stacklevel=1, - ) - onemath = True - if onemkl_interfaces_dir is not None: - warnings.warn( - "Using 'onemkl_interfaces_dir' is deprecated. Please use 'onemath_dir' instead.", - DeprecationWarning, - stacklevel=1, - ) - onemath_dir = onemkl_interfaces_dir - - if target_cuda is not None: - if not target_cuda.strip(): - raise ValueError( - "--target-cuda can not be an empty string. " - "Use --target-cuda= or --target-cuda" - ) - cmake_args += [ - f"-DDPNP_TARGET_CUDA={target_cuda}", - ] - # Always builds using oneMath for the cuda target - onemath = True - - if target_hip is not None: - if not target_hip.strip(): - raise ValueError( - "--target-hip requires an architecture (e.g., gfx90a)" - ) - cmake_args += [ - f"-DHIP_TARGETS={target_hip}", - ] - # Always builds using oneMath for the hip target - onemath = True - - if onemath: - cmake_args += [ - "-DDPNP_USE_ONEMATH=ON", - ] - - if onemath_dir: - cmake_args += [ - f"-DDPNP_ONEMATH_DIR={onemath_dir}", - ] - elif onemath_dir: - raise RuntimeError("--onemath-dir option is not supported") - - subprocess.check_call( - cmake_args, shell=False, cwd=setup_dir, env=os.environ +from _build_helper import ( + build_extension, + clean_build_dir, + err, + get_dpctl_cmake_dir, + install_editable, + log_cmake_args, + make_cmake_args, + resolve_compilers, + resolve_onemath, +) + + +def parse_args(): + p = argparse.ArgumentParser(description="Local dpnp build driver") + + # compiler and oneAPI relating options + p.add_argument( + "--c-compiler", + type=str, + default=None, + help="Path or name of C compiler", ) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser( - description="Driver to build dpnp for in-place installation" + p.add_argument( + "--cxx-compiler", + type=str, + default=None, + help="Path or name of C++ compiler", ) - driver = parser.add_argument_group(title="Coverage driver arguments") - driver.add_argument("--c-compiler", help="Name of C compiler", default=None) - driver.add_argument( - "--cxx-compiler", help="Name of C++ compiler", default=None + p.add_argument( + "--compiler-root", + type=str, + default=None, + help="Path to compiler installation root", ) - driver.add_argument( + p.add_argument( "--oneapi", - help="Set if using one-API installation", dest="oneapi", action="store_true", + help="Use default oneAPI compiler layout", ) - driver.add_argument( - "--debug", - default="Release", - const="Debug", - action="store_const", - help="Set the compilation mode to debugging", + + # CMake relating options + p.add_argument( + "--generator", type=str, default="Ninja", help="CMake generator" ) - driver.add_argument( - "--compiler-root", + p.add_argument( + "--cmake-executable", type=str, - help="Path to compiler home directory", default=None, + help="Path to CMake executable used by build", ) - driver.add_argument( - "--cmake-executable", + p.add_argument( + "--cmake-opts", type=str, - help="Path to cmake executable", - default=None, + default="", + help="Additional options to pass directly to CMake", ) - driver.add_argument( + p.add_argument( + "--debug", + dest="build_type", + const="Debug", + action="store_const", + default="Release", + help="Set build type to Debug (defaults to Release)", + ) + p.add_argument( "--verbose", - help="Build using vebose makefile mode", dest="verbose", action="store_true", + help="Enable verbose makefile output", ) - driver.add_argument( - "--cmake-opts", - help="Channels through additional cmake options", - dest="cmake_opts", - default="", - type=str, - ) - driver.add_argument( + + # platform target relating options + p.add_argument( "--target-cuda", nargs="?", const="ON", - help="Enable CUDA target for build; " - "optionally specify architecture (e.g., --target-cuda=sm_80)", default=None, - type=str, + help="Enable CUDA build. Architecture is optional to specify (e.g., --target-cuda=sm_80).", ) - driver.add_argument( + p.add_argument( "--target-hip", required=False, - help="Enable HIP target for build. " - "Must specify HIP architecture (e.g., --target-hip=gfx90a)", type=str, + help="Enable HIP backend. Architecture required to be specified (e.g., --target-hip=gfx90a).", ) - driver.add_argument( + + # oneMath relating options + p.add_argument( "--onemkl_interfaces", help="(DEPRECATED) Build using oneMath", dest="onemkl_interfaces", action="store_true", ) - driver.add_argument( + p.add_argument( "--onemkl_interfaces_dir", help="(DEPRECATED) Local directory with source of oneMath", dest="onemkl_interfaces_dir", default=None, type=str, ) - driver.add_argument( + p.add_argument( "--onemath", help="Build using oneMath", dest="onemath", action="store_true", ) - driver.add_argument( + p.add_argument( "--onemath-dir", help="Local directory with source of oneMath", dest="onemath_dir", default=None, type=str, ) - args = parser.parse_args() - args_to_validate = [ - "c_compiler", - "cxx_compiler", - "compiler_root", - ] + # build relating options + p.add_argument( + "--clean", + action="store_true", + help="Remove build dir before rebuild", + ) + p.add_argument( + "--skip-editable", + action="store_true", + help="Skip pip editable install step", + ) - if args.oneapi or ( - args.c_compiler is None - and args.cxx_compiler is None - and args.compiler_root is None - ): - args.c_compiler = "icx" - args.cxx_compiler = "icpx" if "linux" in sys.platform else "icx" - args.compiler_root = None - else: - cr = args.compiler_root - if isinstance(cr, str) and os.path.exists(cr): - if args.c_compiler is None: - args.c_compiler = "icx" - if args.cxx_compiler is None: - args.cxx_compiler = "icpx" if "linux" in sys.platform else "icx" - else: - raise RuntimeError( - "Option 'compiler-root' must be provided when " - "using non-default DPC++ layout." - ) - args_to_validate = [ - "c_compiler", - "cxx_compiler", - ] - for p in args_to_validate: - arg = getattr(args, p) - assert isinstance(arg, str) - if not os.path.exists(arg): - arg2 = os.path.join(cr, arg) - if os.path.exists(arg2): - arg = arg2 - setattr(args, p, arg) - if not os.path.exists(arg): - opt_name = p.replace("_", "-") - raise RuntimeError(f"Option {opt_name} value {arg} must exist.") + return p.parse_args() - run( - use_oneapi=args.oneapi, - build_type=args.debug, - c_compiler=args.c_compiler, - cxx_compiler=args.cxx_compiler, - compiler_root=args.compiler_root, - cmake_executable=args.cmake_executable, + +def main(): + if sys.platform not in ["cygwin", "win32", "linux"]: + err(f"{sys.platform} not supported", "build_locally") + + args = parse_args() + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + c_compiler, cxx_compiler = resolve_compilers( + args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root + ) + + dpctl_cmake_dir = get_dpctl_cmake_dir() + print(f"[build_locally] Found DPCTL CMake dir: {dpctl_cmake_dir}") + + onemath, onemath_dir = resolve_onemath( + args.onemath, + args.onemath_dir, + args.target_cuda, + args.target_hip, + args.onemkl_interfaces, + args.onemkl_interfaces_dir, + ) + + # clean build dir if --clean set + if args.clean: + clean_build_dir(setup_dir) + + cmake_args = make_cmake_args( + c_compiler=c_compiler, + cxx_compiler=cxx_compiler, + dpctl_cmake_dir=dpctl_cmake_dir, + onemath=onemath, + onemath_dir=onemath_dir, verbose=args.verbose, - cmake_opts=args.cmake_opts, - target_cuda=args.target_cuda, - target_hip=args.target_hip, - onemkl_interfaces=args.onemkl_interfaces, - onemkl_interfaces_dir=args.onemkl_interfaces_dir, - onemath=args.onemath, - onemath_dir=args.onemath_dir, + other_opts=args.cmake_opts, ) + + # handle architecture conflicts + if args.target_hip is not None and not args.target_hip.strip(): + err("--target-hip requires an explicit architecture", "build_locally") + + # CUDA/HIP targets + if args.target_cuda: + cmake_args += [f"-DDPNP_TARGET_CUDA={args.target_cuda}"] + if args.target_hip: + cmake_args += [f"-DDPNP_TARGET_HIP={args.target_hip}"] + + log_cmake_args(cmake_args, "build_locally") + + print("[build_locally] Building extensions in-place...") + + env = os.environ.copy() + if args.oneapi and "DPL_ROOT" in env: + env["DPL_ROOT_HINT"] = env["DPL_ROOT"] + + build_extension( + setup_dir, + env, + cmake_args, + cmake_executable=args.cmake_executable, + generator=args.generator, + build_type=args.build_type, + ) + if not args.skip_editable: + install_editable(setup_dir, env) + else: + print("[build_locally] Skipping editable install (--skip-editable)") + + print("[build_locally] Build complete") + + +if __name__ == "__main__": + main() diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index a624bd570e2d..a1ac6d5d2ef5 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -26,67 +26,174 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** +import argparse import os import subprocess import sys +from _build_helper import ( + build_extension, + capture_cmd_output, + clean_build_dir, + err, + get_dpctl_cmake_dir, + install_editable, + log_cmake_args, + make_cmake_args, + resolve_compilers, + run, +) -def run( - use_oneapi=True, - c_compiler=None, - cxx_compiler=None, - compiler_root=None, - bin_llvm=None, - pytest_opts="", - verbose=False, -): - IS_LIN = False - - if "linux" in sys.platform: - IS_LIN = True - elif sys.platform in ["win32", "cygwin"]: - pass + +def parse_args(): + p = argparse.ArgumentParser(description="Build dpnp and generate coverage") + + # compiler and oneAPI relating options + p.add_argument( + "--c-compiler", default=None, help="Path or name of C compiler" + ) + p.add_argument( + "--cxx-compiler", default=None, help="Path or name of C++ compiler" + ) + p.add_argument( + "--compiler-root", + type=str, + default=None, + help="Path to compiler installation root", + ) + p.add_argument( + "--oneapi", + dest="oneapi", + action="store_true", + help="Use default oneAPI compiler layout", + ) + p.add_argument( + "--bin-llvm", + type=str, + help="Path to folder where llvm-cov/llvm-profdata can be found", + ) + + # CMake relating options + p.add_argument( + "--generator", type=str, default="Ninja", help="CMake generator" + ) + p.add_argument( + "--cmake-executable", + type=str, + default=None, + help="Path to CMake executable used by build", + ) + + p.add_argument( + "--cmake-opts", + type=str, + default="", + help="Additional options to pass directly to CMake", + ) + p.add_argument( + "--verbose", + dest="verbose", + action="store_true", + help="Enable verbose makefile output", + ) + + # test relating options + p.add_argument( + "--skip-pytest", + dest="run_pytest", + action="store_false", + help="Skip running pytest and coverage generation", + ) + p.add_argument( + "--pytest-opts", + help="Channels through additional pytest options", + dest="pytest_opts", + default="", + type=str, + ) + + # build relating options + p.add_argument( + "--clean", + action="store_true", + help="Remove build dir before rebuild (default: False)", + ) + + return p.parse_args() + + +def find_bin_llvm(compiler): + if os.path.isabs(compiler): + bin_dir = os.path.dirname(compiler) else: - raise AssertionError(sys.platform + " not supported") + compiler_path = capture_cmd_output(["which", compiler]) + if not compiler_path: + raise RuntimeError(f"Compiler {compiler} not found in PATH") + bin_dir = os.path.dirname(compiler_path) - if not IS_LIN: - raise RuntimeError( - "This scripts only supports coverage collection on Linux" - ) + compiler_dir = os.path.join(bin_dir, "compiler") + if os.path.exists(compiler_dir): + bin_llvm = compiler_dir + else: + bin_dir = os.path.dirname(bin_dir) + bin_llvm = os.path.join(bin_dir, "bin-llvm") + + if not os.path.exists(bin_llvm): + raise RuntimeError(f"--bin-llvm value {bin_llvm} not found") + return bin_llvm + + +def main(): + is_linux = "linux" in sys.platform + if not is_linux: + err(f"{sys.platform} not supported", "gen_coverage") + args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - cmake_args = [ - sys.executable, - "setup.py", - "develop", - "--generator=Ninja", - "--", - "-DCMAKE_C_COMPILER:PATH=" + c_compiler, - "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, - "-DDPNP_GENERATE_COVERAGE=ON", - ] - - env = {} - if bin_llvm: - env = { - "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), - "LLVM_TOOLS_HOME": bin_llvm, - } + c_compiler, cxx_compiler = resolve_compilers( + args.oneapi, + args.c_compiler, + args.cxx_compiler, + args.compiler_root, + ) - # extend with global environment variables - env.update({k: v for k, v in os.environ.items() if k != "PATH"}) + dpctl_cmake_dir = get_dpctl_cmake_dir() + print(f"[gen_coverage] Found DPCTL CMake dir: {dpctl_cmake_dir}") - if verbose: - cmake_args += [ - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", - ] + if args.clean: + clean_build_dir(setup_dir) + + cmake_args = make_cmake_args( + c_compiler=c_compiler, + cxx_compiler=cxx_compiler, + dpctl_cmake_dir=dpctl_cmake_dir, + verbose=args.verbose, + ) + cmake_args.append("-DDPNP_GENERATE_COVERAGE=ON") - subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + env = os.environ.copy() + + bin_llvm = find_bin_llvm(c_compiler) + if bin_llvm: + env["PATH"] = ":".join((env.get("PATH", ""), bin_llvm)) + env["LLVM_TOOLS_HOME"] = bin_llvm + + log_cmake_args(cmake_args, "gen_coverage") + + build_extension( + setup_dir, + env, + cmake_args, + cmake_executable=args.cmake_executable, + generator=args.generator, + build_type="Coverage", + ) + install_editable(setup_dir, env) - env["LLVM_PROFILE_FILE"] = "dpnp_pytest.profraw" - subprocess.check_call( - [ + if args.run_pytest: + env["LLVM_PROFILE_FILE"] = "dpnp_pytest.profraw" + pytest_cmd = [ "pytest", "-q", "-ra", @@ -99,133 +206,66 @@ def run( "--cov-report=lcov:coverage-python.lcov", "--pyargs", "dpnp", - *pytest_opts.split(), - ], - cwd=setup_dir, - shell=False, - env=env, - ) - - def find_objects(): - objects = [] - dpnp_path = os.getcwd() - search_path = os.path.join(dpnp_path, "dpnp") - for root, _, files in os.walk(search_path): - for file in files: - if ( - file.endswith("_c.so") - or root.find("extensions") != -1 - and file.find("_impl.cpython") != -1 - ): - objects.extend(["-object", os.path.join(root, file)]) - return objects - - objects = find_objects() - instr_profile_fn = "dpnp_pytest.profdata" - # generate instrumentation profile data - subprocess.check_call( - [ - os.path.join(bin_llvm, "llvm-profdata"), - "merge", - "-sparse", - env["LLVM_PROFILE_FILE"], - "-o", - instr_profile_fn, + *args.pytest_opts.split(), ] - ) + run(pytest_cmd, env=env, cwd=setup_dir) - # export lcov - with open("coverage-cpp.lcov", "w") as fh: - subprocess.check_call( + def find_objects(): + objects = [] + dpnp_path = os.getcwd() + search_path = os.path.join(dpnp_path, "dpnp") + for root, _, files in os.walk(search_path): + for file in files: + if ( + file.endswith("_c.so") + or root.find("extensions") != -1 + and file.find("_impl.cpython") != -1 + ): + objects.extend(["-object", os.path.join(root, file)]) + return objects + + objects = find_objects() + instr_profile_fn = "dpnp_pytest.profdata" + + # generate instrumentation profile data + run( [ - os.path.join(bin_llvm, "llvm-cov"), - "export", - "-format=lcov", - "-ignore-filename-regex=/tmp/icpx*", - r"-ignore-filename-regex=.*/backend/kernels/elementwise_functions/.*\.hpp$", - "-instr-profile=" + instr_profile_fn, + os.path.join(bin_llvm, "llvm-profdata"), + "merge", + "-sparse", + env["LLVM_PROFILE_FILE"], + "-o", + instr_profile_fn, ] - + objects - + ["-sources", "dpnp"], - stdout=fh, ) + # export lcov + with open("coverage-cpp.lcov", "w") as fh: + subprocess.check_call( + [ + os.path.join(bin_llvm, "llvm-cov"), + "export", + "-format=lcov", + "-ignore-filename-regex=/tmp/icpx*", + r"-ignore-filename-regex=.*/backend/kernels/elementwise_functions/.*\.hpp$", + "-instr-profile=" + instr_profile_fn, + ] + + objects + + ["-sources", "dpnp"], + cwd=setup_dir, + env=env, + stdout=fh, + ) -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser( - description="Driver to build dpnp and generate coverage" - ) - driver = parser.add_argument_group(title="Coverage driver arguments") - driver.add_argument("--c-compiler", help="Name of C compiler", default=None) - driver.add_argument( - "--cxx-compiler", help="Name of C++ compiler", default=None - ) - driver.add_argument( - "--not-oneapi", - help="Is one-API installation", - dest="oneapi", - action="store_false", - ) - driver.add_argument( - "--compiler-root", type=str, help="Path to compiler home directory" - ) - driver.add_argument( - "--bin-llvm", help="Path to folder where llvm-cov can be found" - ) - driver.add_argument( - "--pytest-opts", - help="Channels through additional pytest options", - dest="pytest_opts", - default="", - type=str, - ) - driver.add_argument( - "--verbose", - help="Build using vebose makefile mode", - dest="verbose", - action="store_true", - ) - args = parser.parse_args() - - if args.oneapi: - args.c_compiler = "icx" - args.cxx_compiler = "icpx" - args.compiler_root = None - icx_path = subprocess.check_output(["which", "icx"]) - bin_dir = os.path.dirname(icx_path) - compiler_dir = os.path.join(bin_dir.decode("utf-8"), "compiler") - if os.path.exists(compiler_dir): - args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "compiler") - else: - bin_dir = os.path.dirname(bin_dir) - args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") - assert os.path.exists(args.bin_llvm) + print("[gen_coverage] Coverage export is completed") else: - args_to_validate = [ - "c_compiler", - "cxx_compiler", - "compiler_root", - "bin_llvm", - ] - for p in args_to_validate: - arg = getattr(args, p, None) - if not isinstance(arg, str): - opt_name = p.replace("_", "-") - raise RuntimeError( - f"Option {opt_name} must be provided is " - "using non-default DPC++ layout" - ) - if not os.path.exists(arg): - raise RuntimeError(f"Path {arg} must exist") - - run( - use_oneapi=args.oneapi, - c_compiler=args.c_compiler, - cxx_compiler=args.cxx_compiler, - compiler_root=args.compiler_root, - bin_llvm=args.bin_llvm, - pytest_opts=args.pytest_opts, - verbose=args.verbose, - ) + print( + "[gen_coverage] Skipping pytest and coverage collection " + "(--skip-pytest)" + ) + + print("[gen_coverage] Done") + + +if __name__ == "__main__": + main()