Got python package to build correctly using only setuputils.py'

Author: Hoyt Koepke

CMakeLists.txt

@@ -500,22 +500,28 @@ FILE(GLOB_RECURSE _tc_core_files
 list(FILTER _tc_core_files EXCLUDE REGEX ".*/model_server/extensions/.*")
 
 
-add_library(Core SHARED ${_tc_core_files})
-target_link_libraries(Core ${TC_DEPENDENCIES} ${TC_EXTERNAL_DEPENDENCIES})
-add_dependencies(Core external_dependencies ${TC_EXTERNAL_DEPENDENCIES})
-# set_property(TARGET Core APPEND_STRING PROPERTY LINK_FLAGS " -undefined dynamic_lookup")
+add_library(TuriCore SHARED ${_tc_core_files})
+target_link_libraries(TuriCore ${TC_DEPENDENCIES} ${TC_EXTERNAL_DEPENDENCIES})
+add_dependencies(TuriCore external_dependencies ${TC_EXTERNAL_DEPENDENCIES})
+# set_property(TARGET TuriCore APPEND_STRING PROPERTY LINK_FLAGS " -undefined dynamic_lookup")
 source_group(TREE ${CMAKE_SOURCE_DIR} FILES ${_tc_core_files})
 
 set(TC_TARGET_DIR "${CMAKE_SOURCE_DIR}/targets/")
 
-INSTALL(TARGETS Core 
+INSTALL(TARGETS TuriCore 
   LIBRARY DESTINATION ${TC_TARGET_DIR}/lib/)
 INSTALL(DIRECTORY src/
   DESTINATION ${TC_TARGET_DIR}/include/
  MESSAGE_NEVER 
- FILES_MATCHING REGEX "^.*\.((h)|(hpp))$")
+ FILES_MATCHING REGEX "^.*\.((h)|(hpp)|(ipp)|(sparsehash/.*))$")
+
 INSTALL(CODE "execute_process( \
     COMMAND ${CMAKE_COMMAND} -E create_symlink \
     ${TC_TARGET_DIR}/include/external/boost/boost_1_68_0/boost \
     ${TC_TARGET_DIR}/include/boost)")
 
+INSTALL(CODE "execute_process( \
+    COMMAND ${CMAKE_COMMAND} -E create_symlink \
+    ${TC_TARGET_DIR}/include/external/sparsehash/ \
+    ${TC_TARGET_DIR}/include/sparsehash)")
+

src/python/setup.py

@@ -11,22 +11,8 @@
 from setuptools import setup, find_packages, Extension
 from setuptools.dist import Distribution
 from setuptools.command.install import install
-
-extensions = [
-    Extension("primes", ["primes.pyx"],
-        include_dirs=[...],
-        libraries=[...],
-        library_dirs=[...]),
-    # Everything but primes.pyx is included here.
-    Extension("*", ["*.pyx"],
-        include_dirs=[...],
-        libraries=[...],
-        library_dirs=[...]),
-]
-setup(
-    name="My hello app",
-    ext_modules=cythonize(extensions),
-)
+from Cython.Build import cythonize
+from distutils.extension import Extension
 
 PACKAGE_NAME="turicreate"
 VERSION='6.0'#{{VERSION_STRING}}
@@ -92,6 +78,7 @@ def run(self):
 
 
 if __name__ == '__main__':
+
     from distutils.util import get_platform
     classifiers=[
         "Development Status :: 5 - Production/Stable",
@@ -111,7 +98,13 @@ def run(self):
         "Topic :: Scientific/Engineering",
         "Topic :: Scientific/Engineering :: Information Analysis",
     ]
+
+    # Get the include directories. 
+    include_dirs = os.environ.get('CPATH', '').split(';')
+    library_dirs = os.environ.get('LD_LIBRARY_PATH', '').split(';')
+
     cur_platform = get_platform()
+    
     if cur_platform.startswith("macosx"):
         classifiers.append("Operating System :: MacOS :: MacOS X")
     elif cur_platform.startswith('linux'):
@@ -149,15 +142,20 @@ def run(self):
         name="turicreate",
         version=VERSION,
 
-        # This distribution contains platform-specific C++ libraries, but they are not
-        # built with distutils. So we must create a dummy Extension object so when we
-        # create a binary file it knows to make it platform-specific.
-        ext_modules=[Extension('turicreate.__dummy', sources = ['dummy.c'])],
-
         author='Apple Inc.',
         author_email='[email protected]',
         cmdclass=dict(install = InstallEngine),
         distclass=BinaryDistribution,
+
+        ext_modules = cythonize([
+            # Everything but primes.pyx is included here.
+            Extension("*", ["turicreate/_cython/*.pyx"],
+                include_dirs=include_dirs,
+                libraries=['TuriCore'],
+                library_dirs=library_dirs,
+                extra_compile_args=['-O3', '-DNDEBUG', '-std=c++11', '-w']),
+        ]),
+
         package_data={
             'turicreate': [
 

src/python/turicreate/_cython/cy_callback.pxd

@@ -3,6 +3,9 @@
 #
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# distutils: language = c++
+
 """
 Provides a translation layer for easing the use of callback
 functions between cython and the c++ layer.

src/python/turicreate/_cython/cy_callback.pyx

@@ -3,6 +3,9 @@
 #
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# distutils: language = c++
+
 from libcpp.string cimport string
 import traceback
 

src/python/turicreate/_cython/cy_cpp_utils.pxd

@@ -3,6 +3,9 @@
 #
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# distutils: language = c++
+
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 from libcpp.map cimport map

src/python/turicreate/_cython/cy_cpp_utils.pyx

@@ -4,6 +4,9 @@
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 
+# distutils: language = c++
+
+
 from libcpp.string cimport string
 from cpython.version cimport PY_MAJOR_VERSION
 

src/python/turicreate/_cython/cy_dataframe.pxd

@@ -3,6 +3,9 @@
 #
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# distutils: language = c++
+
 from libcpp.vector cimport vector
 from libcpp.string cimport string
 from libcpp.map cimport map

src/python/turicreate/_cython/cy_dataframe.pyx

@@ -3,6 +3,9 @@
 #
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# distutils: language = c++
+
 from .cy_flexible_type cimport common_typed_flex_list_from_iterable
 from .cy_flexible_type cimport pylist_from_flex_list
 from .cy_flexible_type cimport pytype_from_flex_type_enum

src/python/turicreate/_cython/cy_flexible_type.pxd

@@ -4,6 +4,9 @@
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 
+# distutils: language = c++
+
+
 #cython libcpp types
 from libcpp.vector cimport vector
 from libcpp.string cimport string

src/python/turicreate/_cython/cy_flexible_type.pyx

@@ -3,6 +3,9 @@
 #
 # Use of this source code is governed by a BSD-3-clause license that can
 # be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# distutils: language = c++
+
 '''
 There are essentially three tasks that need to be done in flexible_type translation;
 each of these are implemented in this file.