setup.py for building C++ backend (#168)

Co-authored-by: etotmeni <[email protected]>

Author: 1e-to

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 - Device descriptors "max_compute_units", "max_work_item_dimensions", "max_work_item_sizes", "max_work_group_size", "max_num_sub_groups" and "aspects" for int64 atomics inside dpctl C API and inside the dpctl.SyclDevice class.
 - MemoryUSM* classes moved to `dpctl.memory` module, added support for aligned allocation, added support for `prefetch` and `mem_advise` (sychronous) methods, implemented `copy_to_host`, `copy_from_host` and `copy_from_device` methods, pickling support, and zero-copy interoperability with Python objects which implement `__sycl_usm_array_inerface__` protocol.
 - Helper scripts to generate API documentation for both C API and Python.
+- setup.py builds C++ backend for develop and install commands.
 
 ### Fixed
 - Compiler warnings when building libDPPLSyclInterface and the Cython extensions.

README.md

@@ -36,6 +36,20 @@ conda build --croot=C:/tmp conda-recipe
 conda install dpctl
 ```
 
+Build and Install with setuptools
+=================================
+dpCtl relies on DPC++ runtime. With Intel oneAPI installed you should activate it.
+
+For install:
+```cmd
+python setup.py install
+```
+
+For development:
+```cmd
+python setup.py develop
+```
+
 Using dpCtl
 ===========
 dpCtl relies on DPC++ runtime. With Intel oneAPI installed you should activate it.

conda-recipe/bld.bat

@@ -3,45 +3,7 @@ IF %ERRORLEVEL% NEQ 0 (
 	echo "oneAPI compiler activation failed"
 	exit /b 1
 )
-REM conda uses %ERRORLEVEL% but FPGA scripts can set it. So it should be reseted.
-set ERRORLEVEL=
-
-set "CC=clang-cl.exe"
-set "CXX=dpcpp.exe"
-
-rmdir /S /Q build_cmake
-mkdir build_cmake
-cd build_cmake
-
-set "DPCPP_ROOT=%ONEAPI_ROOT%\compiler\latest\windows"
-set "INSTALL_PREFIX=%cd%\..\install"
-
-rmdir /S /Q "%INSTALL_PREFIX%"
-
-cmake -G Ninja ^
-    -DCMAKE_BUILD_TYPE=Release ^
-    "-DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX%" ^
-    "-DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX%" ^
-    "-DDPCPP_ROOT=%DPCPP_ROOT%" ^
-    "%SRC_DIR%\backends"
-IF %ERRORLEVEL% NEQ 0 exit /b 1
-
-ninja -n
-ninja install
-IF %ERRORLEVEL% NEQ 0 exit /b 1
-
-cd ..
-xcopy install\lib\*.lib dpctl /E /Y
-xcopy install\bin\*.dll dpctl /E /Y
-
-mkdir dpctl\include
-xcopy backends\include dpctl\include /E /Y
-
-
-REM required by _sycl_core(dpctl)
-set "DPPL_SYCL_INTERFACE_LIBDIR=dpctl"
-set "DPPL_SYCL_INTERFACE_INCLDIR=dpctl\include"
 
 "%PYTHON%" setup.py clean --all
-"%PYTHON%" setup.py build install
+"%PYTHON%" setup.py install
 IF %ERRORLEVEL% NEQ 0 exit /b 1

conda-recipe/build.sh

@@ -4,50 +4,10 @@
 if [ ! -z "${ONEAPI_ROOT}" ]; then
     # Suppress error b/c it could fail on Ubuntu 18.04
     source ${ONEAPI_ROOT}/compiler/latest/env/vars.sh || true
-    export CC=clang
-    export CXX=clang++
 else
     echo "DPCPP is needed to build DPPL. Abort!"
     exit 1
 fi
 
-rm -rf build_cmake
-mkdir build_cmake
-pushd build_cmake
-
-INSTALL_PREFIX=`pwd`/../install
-rm -rf ${INSTALL_PREFIX}
-
-PYTHON_INC=`${PYTHON} -c "import distutils.sysconfig;                  \
-                        print(distutils.sysconfig.get_python_inc())"`
-NUMPY_INC=`${PYTHON} -c "import numpy; print(numpy.get_include())"`
-DPCPP_ROOT=${ONEAPI_ROOT}/compiler/latest/linux/
-
-cmake                                                       \
-    -DCMAKE_BUILD_TYPE=Release                              \
-    -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}                \
-    -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX}                   \
-    -DDPCPP_ROOT=${DPCPP_ROOT}                              \
-    -DPYTHON_INCLUDE_DIR=${PYTHON_INC}                      \
-    -DNUMPY_INCLUDE_DIR=${NUMPY_INC}                        \
-    ../backends
-
-make -j 4 && make install
-
-popd
-cp install/lib/*.so dpctl/
-
-mkdir -p dpctl/include
-cp -r backends/include/* dpctl/include
-
-
-# required by dpctl.sycl_core
-export DPPL_SYCL_INTERFACE_LIBDIR=dpctl
-export DPPL_SYCL_INTERFACE_INCLDIR=dpctl/include
-
-
-# FIXME: How to pass this using setup.py? This flags is needed when
-# dpcpp compiles the generated cpp file.
-export CFLAGS="-fPIC -O3 ${CFLAGS}"
 ${PYTHON} setup.py clean --all
-${PYTHON} setup.py build install
+${PYTHON} setup.py install

scripts/build_backend.py

@@ -0,0 +1,82 @@
+import os
+import sys
+import subprocess
+import shutil
+import glob
+
+IS_WIN = False
+IS_LIN = False
+
+if "linux" in sys.platform:
+    IS_LIN = True
+elif sys.platform in ["win32", "cygwin"]:
+    IS_WIN = True
+else:
+    assert False, sys.platform + " not supported"
+
+ONEAPI_ROOT = os.environ.get("ONEAPI_ROOT")
+
+if IS_LIN:
+    DPCPP_ROOT = os.path.join(ONEAPI_ROOT, "compiler/latest/linux")
+if IS_WIN:
+    DPCPP_ROOT = os.path.join(ONEAPI_ROOT, "compiler\latest\windows")
+
+dpctl_dir = os.getcwd()
+build_cmake_dir = os.path.join(dpctl_dir, "build_cmake")
+if os.path.exists(build_cmake_dir):
+    shutil.rmtree(build_cmake_dir)
+os.mkdir(build_cmake_dir)
+os.chdir(build_cmake_dir)
+
+INSTALL_PREFIX = os.path.join(dpctl_dir, "install")
+if os.path.exists(INSTALL_PREFIX):
+    shutil.rmtree(INSTALL_PREFIX)
+
+backends_dir = os.path.join(dpctl_dir, "backends")
+
+if IS_LIN:
+    cmake_args = [
+        "cmake",
+        "-DCMAKE_BUILD_TYPE=Release",
+        "-DCMAKE_INSTALL_PREFIX=" + INSTALL_PREFIX,
+        "-DCMAKE_PREFIX_PATH=" + INSTALL_PREFIX,
+        "-DDPCPP_ROOT=" + DPCPP_ROOT,
+        "-DCMAKE_C_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "clang"),
+        "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "clang++"),
+        backends_dir,
+    ]
+    subprocess.check_call(cmake_args, stderr=subprocess.STDOUT, shell=False)
+    subprocess.check_call(["make", "-j", "4"])
+    subprocess.check_call(["make", "install"])
+
+    os.chdir(dpctl_dir)
+    for file in glob.glob(os.path.join(dpctl_dir, "install", "lib", "*.so")):
+        shutil.copy(file, os.path.join(dpctl_dir, "dpctl"))
+
+if IS_WIN:
+    cmake_args = [
+        "cmake",
+        "-G",
+        "Ninja",
+        "-DCMAKE_BUILD_TYPE=Release",
+        "-DCMAKE_INSTALL_PREFIX=" + INSTALL_PREFIX,
+        "-DCMAKE_PREFIX_PATH=" + INSTALL_PREFIX,
+        "-DDPCPP_ROOT=" + DPCPP_ROOT,
+        backends_dir,
+    ]
+    subprocess.check_call(cmake_args, stderr=subprocess.STDOUT, shell=True)
+    subprocess.check_call(["ninja", "-n"])
+    subprocess.check_call(["ninja", "install"])
+
+    os.chdir(dpctl_dir)
+    for file in glob.glob(os.path.join(dpctl_dir, "install", "lib", "*.lib")):
+        shutil.copy(file, os.path.join(dpctl_dir, "dpctl"))
+
+    for file in glob.glob(os.path.join(dpctl_dir, "install", "bin", "*.dll")):
+        shutil.copy(file, os.path.join(dpctl_dir, "dpctl"))
+
+include_dir = os.path.join(dpctl_dir, "dpctl", "include")
+if os.path.exists(include_dir):
+    shutil.rmtree(include_dir)
+
+shutil.copytree(os.path.join(dpctl_dir, "backends", "include"), include_dir)

setup.py

@@ -25,7 +25,10 @@
 import os.path
 import sys
 import versioneer
+import subprocess
 
+import setuptools.command.install as orig_install
+import setuptools.command.develop as orig_develop
 from setuptools import setup, Extension, find_packages
 from Cython.Build import cythonize
 
@@ -48,6 +51,20 @@
 else:
     assert False, sys.platform + " not supported"
 
+if IS_LIN:
+    DPCPP_ROOT = os.environ["ONEAPI_ROOT"] + "/compiler/latest/linux"
+    os.environ["CC"] = DPCPP_ROOT + "/bin/clang"
+    os.environ["CXX"] = DPCPP_ROOT + "/bin/clang++"
+    os.environ["DPPL_SYCL_INTERFACE_LIBDIR"] = "dpctl"
+    os.environ["DPPL_SYCL_INTERFACE_INCLDIR"] = "dpctl/include"
+    os.environ["CFLAGS"] = "-fPIC"
+
+elif IS_WIN:
+    os.environ["CC"] = "clang-cl.exe"
+    os.environ["CXX"] = "dpcpp.exe"
+    os.environ["DPPL_SYCL_INTERFACE_LIBDIR"] = "dpctl"
+    os.environ["DPPL_SYCL_INTERFACE_INCLDIR"] = "dpctl\include"
+
 dppl_sycl_interface_lib = os.environ["DPPL_SYCL_INTERFACE_LIBDIR"]
 dppl_sycl_interface_include = os.environ["DPPL_SYCL_INTERFACE_INCLDIR"]
 sycl_lib = os.environ["ONEAPI_ROOT"] + "\compiler\latest\windows\lib"
@@ -99,6 +116,11 @@ def get_suppressed_warning_flags():
         return []
 
 
+def build_backend():
+    build_script = os.path.join(os.getcwd(), "scripts", "build_backend.py")
+    subprocess.check_call([sys.executable, build_script])
+
+
 def extensions():
     # Security flags
     eca = get_sdl_cflags()
@@ -161,10 +183,29 @@ def extensions():
     return exts
 
 
+class install(orig_install.install):
+    def run(self):
+        build_backend()
+        return super().run()
+
+
+class develop(orig_develop.develop):
+    def run(self):
+        build_backend()
+        return super().run()
+
+
+def _get_cmdclass():
+    cmdclass = versioneer.get_cmdclass()
+    cmdclass["install"] = install
+    cmdclass["develop"] = develop
+    return cmdclass
+
+
 setup(
     name="dpctl",
     version=versioneer.get_version(),
-    cmdclass=versioneer.get_cmdclass(),
+    cmdclass=_get_cmdclass(),
     description="A lightweight Python wrapper for a subset of OpenCL and SYCL.",
     license="Apache 2.0",
     author="Intel Corporation",