PYTHON-3047 Clean up handling of pyarrow library (#39)

blink1073 · web-flow · commit a5fc0ba83714 · 2021-12-16T14:02:36.000-06:00
diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -4,6 +4,10 @@ on:
   push:
   pull_request:
 
+defaults:  
+  run:  
+    working-directory: ./bindings/python
+
 jobs:
   build:
     # supercharge/mongodb-github-action requires containers so we don't test other platforms
@@ -27,21 +31,17 @@ jobs:
       - name: Install libbson
         run: |
           LIBBSON_INSTALL_DIR=$(pwd)/libbson ./build-libbson.sh
-        working-directory: ./bindings/python
       - name: Install Python dependencies
         run: |
-          python -m pip install -U pip wheel
-          python -m pip install -r requirements/test.txt
-          python -c 'import pyarrow as pa; pa.create_library_symlinks()'
-        working-directory: ./bindings/python
+          python -m pip install -U pip
       - name: Install pymongoarrow
         run: |
-          # Use pip install to pull in install_requires dependencies
-          LIBBSON_INSTALL_DIR=$(pwd)/libbson python -m pip install -e .
-          # Re-compile to see compiler/linker output in the logs
-          LIBBSON_INSTALL_DIR=$(pwd)/libbson python setup.py build_ext --inplace --force
-        working-directory: ./bindings/python
+          # Install the library
+          LIBBSON_INSTALL_DIR=$(pwd)/libbson python -m pip install -vvv -e ".[test]"
       - name: Run tests
         run: |
           LD_LIBRARY_PATH=$(pwd)/libbson/lib python -m unittest discover test -v
-        working-directory: ./bindings/python
+      - name: Check the manifest
+        run: |
+          pip install check-manifest
+          check-manifest -v
diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore
@@ -5,6 +5,7 @@ __pycache__/
 
 # C extensions
 *.so
+*.dylib
 
 # Distribution / packaging
 .Python
diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in
@@ -1,14 +1,21 @@
 include README.rst
 include LICENSE
 include build-libbson.sh
-include release.sh
+include pyproject.toml
+
+exclude THIRD-PARTY-NOTICES
+exclude release.sh
+exclude addtags.py
+exclude benchmark.py
 
 graft pymongoarrow
 
-prune test
+recursive-exclude test *
+recursive-exclude docs *
 
 global-exclude *.cpp
 global-exclude *.dylib
+global-exclude *.so.*
 global-exclude *.so
 global-exclude *.pyc
 global-exclude .git*
diff --git a/bindings/python/docs/source/developer/installation.rst b/bindings/python/docs/source/developer/installation.rst
@@ -41,23 +41,22 @@ On macOS, users can install the latest ``libbson`` via Homebrew::
 
   $ brew install mongo-c-driver
 
+Alternatively, you can use the provided `build-libbson.sh` script to build it::
+
+  $ LIBBSON_INSTALL_DIR=$(pwd)/libbson ./build-libbson.sh
+
 
 Build
 -----
 
-In the previously created virtualenv, we first install all build and test dependencies
-of PyMongoArrow::
+In the previously created virtualenv, install PyMongoArrow and its test dependencies in editable mode::
 
-  (pymongoarrow) $ pip install -r requirements/test.txt
+  (pymongoarrow) $ pip install -v -e ".[test]"
 
-We also need the pyarrow library symlinks, which can be created as follows::
- 
-  (pymongoarrow) $ python -c 'import pyarrow as pa; pa.create_library_symlinks()'
+If you built libbson using the `build-libbson` script then use the same `LIBBSON_INSTALL_DIR` as above:
 
-We can now install ``pymongoarrow`` in **development mode** as follows::
+  (pymongoarrow) $ LIBBSON_INSTALL_DIR=$(pwd)/libbson pip install -v -e ".[test]"
 
-  (pymongoarrow) $ python setup.py build_ext --inplace
-  (pymongoarrow) $ python setup.py develop
 
 Test
 ----
diff --git a/bindings/python/pymongoarrow/__init__.py b/bindings/python/pymongoarrow/__init__.py
@@ -15,9 +15,10 @@
 # We must import pyarrow before attempting to load the Cython module.
 import pyarrow
 
-from pymongoarrow.lib import libbson_version
 from pymongoarrow.version import __version__, _MIN_LIBBSON_VERSION
-
+# This line must come second so setuptools can parse the __version__
+# above without having a built application.
+from pymongoarrow.lib import libbson_version
 
 try:
     from pkg_resources import parse_version as _parse_version
diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml
@@ -0,0 +1,15 @@
+[build-system]
+requires = [
+    "setuptools>=47.9",
+    "wheel>=0.37",
+    "cython>=0.29",
+    # Must be kept in sync with the `install_requires` in `setup.cfg`
+    "pyarrow>=6.0.0,<6.1.0",
+]
+
+[tool.check-manifest]
+ignore = [
+    "pymongoarrow/*.so*.*", 
+    "pymongoarrow/*.so*", 
+    "pymongoarrow/*.dylib"
+]
diff --git a/bindings/python/release.sh b/bindings/python/release.sh
@@ -38,14 +38,10 @@ $PYTHON --version
 cp $LIBBSON_INSTALL_DIR/lib*/$LIBBSON_SO "$(pwd)/pymongoarrow/"
 
 # Install build dependencies
-$PYTHON -m pip install -U pip wheel
-$PYTHON -m pip install -r requirements/build.txt
-
-# https://arrow.apache.org/docs/python/extending.html#building-extensions-against-pypi-wheels
-$PYTHON -c "import pyarrow; pyarrow.create_library_symlinks()"
+$PYTHON -m pip install -U pip build
 
 # Build wheels in $(pwd)/dist/*.whl
-LIBBSON_INSTALL_DIR="$LIBBSON_INSTALL_DIR" $PYTHON setup.py bdist_wheel
+LIBBSON_INSTALL_DIR="$LIBBSON_INSTALL_DIR" $PYTHON -m build --wheel .
 
 # Run auditwheel repair to set platform tags on Linux
 if [ "Linux" = "$(uname -s)" ]
diff --git a/bindings/python/requirements/build.txt b/bindings/python/requirements/build.txt
diff --git a/bindings/python/requirements/test.txt b/bindings/python/requirements/test.txt
diff --git a/bindings/python/setup.cfg b/bindings/python/setup.cfg
@@ -0,0 +1,50 @@
+
+[metadata]
+name = pymongoarrow
+version = attr: pymongoarrow.version.__version__
+description = "Tools for using NumPy, Pandas and PyArrow with MongoDB"
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+license = Apache License, Version 2.0
+author = Prashant Mital
+maintainer = MongoDB, Inc.
+url = https://github.com/mongodb-labs/mongo-arrow/tree/main/bindings/python
+platforms = Linux, Mac OS X
+keywords = mongo, mongodb, pymongo, arrow, bson, numpy, pandas
+classifiers =
+    Development Status :: 3 - Alpha
+    Intended Audience :: Developers
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: Apache Software License
+    Operating System :: MacOS :: MacOS X
+    Operating System :: POSIX
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: Implementation :: CPython
+    Topic :: Database
+
+[options]
+zip_safe = False
+include_package_data = True
+packages = find:
+python_requires = >=3.6
+install_requires =
+    # keep versions in sync with pyproject.toml "requires"
+    pyarrow >=6,<6.1
+    pymongo >=3.11,<5
+
+[options.package_data]
+pymongoarrow = *.pxd, *.pyx, *.pyi, *.so.*, *.dylib
+
+[options.packages.find]
+exclude =
+    test
+    docs
+
+[options.extras_require]
+test = pandas;pytz
diff --git a/bindings/python/setup.py b/bindings/python/setup.py
@@ -1,37 +1,20 @@
-from setuptools import setup, find_packages
-from Cython.Build import cythonize
+import shutil
+from setuptools import setup
 
 import glob
 import os
+import re
 import subprocess
 from sys import platform
 import warnings
 
-import numpy as np
-import pyarrow as pa
 
+HERE = os.path.abspath(os.path.dirname(__file__))
 
 if platform not in ('linux', 'darwin'):
     raise RuntimeError("Unsupported plaform {}".format(platform))
 
 
-# Keep these dependencies synced with those in requirements/*.txt
-PYARROW_DEP = 'pyarrow>=6,<6.1'
-PYMONGO_DEP = 'pymongo>=3.11,<5'
-INSTALL_REQUIRES = [PYARROW_DEP, PYMONGO_DEP]
-SETUP_REQUIRES = ['setuptools>=47', 'cython>=0.29', PYARROW_DEP]
-
-
-def get_pymongoarrow_version():
-    """Single source the version."""
-    version_file = os.path.realpath(os.path.join(
-        os.path.dirname(__file__), 'pymongoarrow', 'version.py'))
-    version = {}
-    with open(version_file) as fp:
-        exec(fp.read(), version)
-    return version['__version__']
-
-
 def query_pkgconfig(cmd):
     status, output = subprocess.getstatusoutput(cmd)
     if status != 0:
@@ -41,6 +24,7 @@ def query_pkgconfig(cmd):
 
 
 def append_libbson_flags(module):
+    pc_path = 'libbson-1.0'
     install_dir = os.environ.get('LIBBSON_INSTALL_DIR')
     if install_dir:
         libdirs = glob.glob(os.path.join(install_dir, "lib*"))
@@ -50,8 +34,10 @@ def append_libbson_flags(module):
             libdir = libdirs[0]
             pc_path = os.path.join(
                 install_dir, libdir, 'pkgconfig', 'libbson-1.0.pc')
-    else:
-        pc_path = 'libbson-1.0'
+
+    lnames = query_pkgconfig("pkg-config --libs-only-l {}".format(pc_path))
+    if not lnames:
+        raise ValueError(f'Could not find "{pc_path}" library')
 
     cflags = query_pkgconfig("pkg-config --cflags {}".format(pc_path))
     if cflags:
@@ -72,18 +58,49 @@ def append_libbson_flags(module):
         module.extra_link_args += ["-Wl,-rpath,$ORIGIN"]
 
     # https://cython.readthedocs.io/en/latest/src/tutorial/external.html#dynamic-linking
-    lnames = query_pkgconfig("pkg-config --libs-only-l {}".format(pc_path)).split()
+    lnames = lnames.split()
     # Strip whitespace to avoid weird linker failures on manylinux images
     libnames = [lname.lstrip('-l').strip() for lname in lnames]
     module.libraries.extend(libnames)
 
 
 def append_arrow_flags(module):
+    import numpy as np
+    import pyarrow as pa
+
     module.include_dirs.append(np.get_include())
     module.include_dirs.append(pa.get_include())
-    module.libraries.extend(pa.get_libraries())
     module.library_dirs.extend(pa.get_library_dirs())
 
+    # Handle the arrow library files manually.
+    # Alternative to using pyarrow.create_library_symlinks().
+    # You can use MONGO_ARROW_LIBDIR to explicitly set the location of the 
+    # arrow libraries (for instance in a conda build).
+    # We first check for an unmodified path to the library,
+    # then look for a library file with a version modifier, e.g. libarrow.600.dylib.
+    arrow_lib = os.environ.get('MONGO_ARROW_LIBDIR', pa.get_library_dirs()[0])
+    if platform == "darwin":
+        exts = ['.dylib', '.*.dylib']
+    elif platform == 'linux':
+        exts = ['.so', '.so.*']
+
+    # Find the appropriate library file and optionally copy it locally.
+    for name in pa.get_libraries():
+        for ext in exts:
+            files = glob.glob(os.path.join(arrow_lib, f'lib{name}{ext}'))
+            if not files:
+                continue
+            path = files[0]
+    
+            # You can use MONGO_NO_COPY_ARROW_LIB to avoid copying the arrow library
+            # files to the build directory (for instance in a conda build).
+            if not os.environ.get("MONGO_NO_COPY_ARROW_LIB", False):
+                build_dir = os.path.join(HERE, 'pymongoarrow')
+                shutil.copy(path, build_dir)
+                path = os.path.join(build_dir, os.path.basename(path))
+            module.extra_link_args.append(path)
+            break
+
     # Arrow's manylinux{2010, 2014} binaries are built with gcc < 4.8 which predates CXX11 ABI
     # - https://uwekorn.com/2019/09/15/how-we-build-apache-arrows-manylinux-wheels.html
     # - https://arrow.apache.org/docs/python/extending.html#example
@@ -93,6 +110,12 @@ def append_arrow_flags(module):
 
 
 def get_extension_modules():
+    # This change is needed in order to allow setuptools to import the
+    # library to obtain metadata information outside of a build environment.
+    try:
+        from Cython.Build import cythonize
+    except ImportError:
+        return []
     modules = cythonize(['pymongoarrow/*.pyx'])
     for module in modules:
         append_libbson_flags(module)
@@ -101,47 +124,4 @@ def get_extension_modules():
     return modules
 
 
-with open('README.rst') as f:
-    LONG_DESCRIPTION = f.read()
-
-
-setup(
-    name='pymongoarrow',
-    packages=find_packages(),
-    zip_safe=False,
-    package_data={
-        "pymongoarrow": ['*.pxd', '*.pyx', '*.pyi', '*.so.*', '*.dylib']},
-    ext_modules=get_extension_modules(),
-    version=get_pymongoarrow_version(),
-    python_requires=">=3.6",
-    install_requires=INSTALL_REQUIRES,
-    setup_requires=SETUP_REQUIRES,
-    tests_require=["pandas", "pytz"],
-    description="Tools for using NumPy, Pandas and PyArrow with MongoDB",
-    long_description=LONG_DESCRIPTION,
-    long_description_content_type='text/x-rst',
-    classifiers=[
-        'Development Status :: 3 - Alpha',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Science/Research',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: Implementation :: CPython',
-        'Topic :: Database'],
-    license='Apache License, Version 2.0',
-    author="Prashant Mital",
-    author_email="mongodb-user@googlegroups.com",
-    maintainer="MongoDB, Inc.",
-    maintainer_email="mongodb-user@googlegroups.com",
-    url="https://github.com/mongodb-labs/mongo-arrow/tree/main/bindings/python",
-    keywords=["mongo", "mongodb", "pymongo", "arrow", "bson",
-              "numpy", "pandas"],
-    test_suite="test"
-)
+setup(ext_modules=get_extension_modules())
