Integrate cibuildwheel commands into pyproject.toml (#13)

wbarnha · maxibor · twmht · web-flow · commit 7ae0187b71ab · 2022-12-22T14:00:04.000-05:00
* Change to relative imports and attempt to make a windows wheel

* Fixes

* Drop zstd

* Last attempt

* commit initial progress for supporting windows

* update pyproject.toml

* update pyproject.toml and revert options.pxd

* simplify build workflows, only use cibuildwheel from hereon

* fix typo

* update cibuildwheel name

* go to rocksdb 6.14.6

Co-authored-by: maxibor &lt;maxime.borry@gmail.com&gt;
Co-authored-by: Ming-Hsuan-Tu &lt;qrnnis2623891@gmail.com&gt;
Co-authored-by: AustEcon &lt;AustEcon0922@gmail.com&gt;
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -19,7 +19,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        rocksdb_ver: ['v6.29.3', 'v6.25.3', 'v6.11.4']
+        rocksdb_ver: ['v6.29.3', 'v6.25.3', 'v6.14.6']
 
     steps:
       - uses: actions/cache@v2
@@ -53,11 +53,13 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         py_ver: ['3.7', '3.8', '3.9', '3.10', '3.11']
-        rocksdb_ver: ['v6.29.3', 'v6.25.3', 'v6.11.4']
+        rocksdb_ver: ['v6.29.3', 'v6.25.3', 'v6.14.6']
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         name: 'Checkout source repository'
+        with:
+            fetch-depth: 0
 
       - uses: actions/setup-python@v4
         name: 'Set up Python ${{ matrix.py_ver }}'
diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml
@@ -16,10 +16,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-#        dist: [bullseye, bookworm]
         dist: [bullseye]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        name: 'Checkout source repository'
+        with:
+            fetch-depth: 0
 
       - name: Install build-dependencies
         # TODO(dato): find out why setup.py links to compression libraries
diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml
@@ -1,27 +1,25 @@
 # vim:ts=2:sw=2:et:ai:sts=2
-name: 'Build distribution'
+name: 'Build and test wheels for release'
 
 on:
-  # Only run when release is created in the master branch
   release:
     types: [created]
     branches:
       - 'master'
 
 jobs:
   build_wheels:
-    name: 'Build wheels'
+    name: 'Build and test wheels'
     runs-on: ${{ matrix.os }}
-    env:
-      LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }}
     strategy:
       matrix:
         os: [ubuntu-latest]
-        rocksdb_ver: ['v6.14.6']
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         name: 'Checkout source repository'
+        with:
+            fetch-depth: 0
 
       - uses: actions/setup-python@v4
         name: 'Set up Python 3.11'
@@ -32,47 +30,25 @@ jobs:
         run: |
           python3 -m pip install cibuildwheel
 
-      - name: 'Build wheels'
+      - name: 'Build and test wheels'
         run: |
           python3 -m cibuildwheel --output-dir dist
-        env:
-          CIBW_MANYLINUX_X86_64_IMAGE: 'manylinux2014'
-          CIBW_ARCHS: auto64
-          CIBW_BUILD: 'cp3*'
-          CIBW_SKIP: '*-musllinux_*'
-          # Install python package and test-deps.
-          CIBW_TEST_REQUIRES: '.[test] pytest'
-          # Use `--pyargs` to interpret parameter as module to import, not as a
-          # path, and do not use `python3 -m pytest`. This way we prevent
-          # importing the module from the current directory instead of the
-          # installed package, and failing when it cannot find the shared
-          # library.
-          CIBW_TEST_COMMAND: 'pytest --pyargs rocksdb'
-          # Avoid re-building the C library in every iteration by testing for
-          # the build directory.
-          CIBW_BEFORE_BUILD: >
-            yum install -y bzip2-devel lz4-devel snappy-devel zlib-devel
-            python3-Cython && (
-              test -d ${{ env.LIBROCKSDB_PATH }} || (
-              git clone https://github.com/facebook/rocksdb --depth 1 --branch ${{ matrix.rocksdb_ver }} ${{ env.LIBROCKSDB_PATH }} &&
-              cd ${{ env.LIBROCKSDB_PATH }} &&
-              CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4
-            )) &&
-            pushd ${{ env.LIBROCKSDB_PATH }} &&
-            make install-shared &&
-            ldconfig &&
-            popd
+
       - uses: actions/upload-artifact@v2
         name: 'Upload build artifacts'
+        if: github.event_name == 'release' && github.event.action == 'created'
         with:
           path: 'dist/*.whl'
 
   build_sdist:
     name: 'Build source distribution'
+    if: github.event_name == 'release' && github.event.action == 'created'
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         name: 'Checkout source repository'
+        with:
+          fetch-depth: 0
 
       - uses: actions/setup-python@v4
         name: 'Set up Python 3.10'
@@ -89,8 +65,6 @@ jobs:
         with:
           path: 'dist/*.tar.gz'
 
-
-
   upload_pypi:
     name: 'Upload packages'
     needs: ['build_wheels', 'build_sdist']
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -1,10 +1,10 @@
 name: Pages
 
 on:
-  push:
+  release:
+    types: [created]
     branches:
-    - master
-    - create-gh-pages
+      - 'master'
 
 jobs:
   build:
diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,6 @@ docs/_build
 *.so
 __pycache__
 rocksdb/_rocksdb.cpp
+*idea
+*html
 
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 ENV SRC /home/tester/src
 ENV DEBIAN_FRONTEND noninteractive
 
diff --git a/docs/index.rst b/docs/index.rst
@@ -11,7 +11,7 @@ Python bindings to the C++ interface of http://rocksdb.org/ using cython::
     db.put(b"a", b"b")
     print(db.get(b"a"))
 
-Tested with python3.8, python3.9, python3.10, and RocksDB version 6.11.4.
+Tested with python3.8, python3.9, python3.10, and RocksDB version 6.14.6.
 
 .. toctree::
     :maxdepth: 2
diff --git a/docs/installation.rst b/docs/installation.rst
@@ -2,7 +2,6 @@ Installing
 ==========
 .. highlight:: bash
 
-
 With distro package and pypi
 ****************************
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,3 +1,52 @@
 [build-system]
-requires = ["setuptools", "wheel", "cython", "pkgconfig"]
+requires = ["setuptools", "wheel", "cython", "pkgconfig", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
+
+[tool.cibuildwheel]
+build-frontend = "build"
+build = "cp3*"
+skip = ["*-win32", "*-manylinux_i686", "*-musllinux_*"]
+archs = ["auto64"]
+test-requires = ['.[test] pytest']
+test-command = ['pytest --pyargs rocksdb']
+
+manylinux-x86_64-image = "manylinux2014"
+
+environment = {ROCKSDB_VER="v6.14.6", LIBROCKSDB_PATH="/opt/rocksdb-$ROCKSDB_VER"}
+
+[tool.cibuildwheel.linux]
+# Avoid re-building the C library in every iteration by testing for the build directory.
+before-build = "yum install -y bzip2-devel lz4-devel snappy-devel zlib-devel python3-Cython && (test -d $LIBROCKSDB_PATH || (  git clone https://github.com/facebook/rocksdb --depth 1 --branch $ROCKSDB_VER $LIBROCKSDB_PATH &&  cd $LIBROCKSDB_PATH &&  CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4  )) && pushd $LIBROCKSDB_PATH && make install-shared &&  ldconfig && popd"
+
+[tool.cibuildwheel.macos]
+environment = {ROCKSDB_VER="v6.14.6", LIBROCKSDB_PATH="/usr/local/opt/rocksdb-$ROCKSDB_VER", REPAIR_LIBRARY_PATH="$LIBROCKS_DB_PATH"}
+before-build = [
+    "brew install bzip2 lz4 snappy zlib cython git",
+    'export LDFLAGS="-L/usr/local/opt/bzip2/lib"',
+    'export CPPFLAGS="-I/usr/local/opt/bzip2/include"',
+    "(test -d $LIBROCKSDB_PATH || (  git clone https://github.com/facebook/rocksdb --depth 1 --branch $ROCKSDB_VER $LIBROCKSDB_PATH &&  cd $LIBROCKSDB_PATH &&  CXXFLAGS='-flto -Os' PORTABLE=1 make shared_lib -j 4  )) && pushd $LIBROCKSDB_PATH && make install-shared &&  update_dyld_shared_cache && popd",
+    "otool -L librocksdb.6.14.dylib",
+]
+repair-wheel-command = """\
+DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} &&
+DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel \
+--require-archs {delocate_archs} -w {dest_dir} -v {wheel}\
+"""
+
+[tool.cibuildwheel.windows]
+before-build = [
+    "vcpkg install zlib:x64-windows",
+    "vcpkg install bzip2:x64-windows",
+    "vcpkg install lz4:x64-windows",
+    "vcpkg install snappy:x64-windows",
+    "vcpkg install rocksdb[bzip2,zlib,lz4,snappy]:x64-windows --recurse",
+    'echo $(python --version) && python -m pip install cython && python -m cython -3 --cplus --fast-fail --annotate rocksdb/_rocksdb.pyx && python ./setup.py build_ext --include-dirs=$(VCPKG_ROCKSDB_INCLUDE_PATH) --library-dirs=$(VCPKG_ROCKSDB_LIB_PATH) --libraries"=shlwapi rpcrt4"',
+]
+
+[tool.cibuildwheel.windows.environment]
+VCPKG_ROCKSDB_INCLUDE_PATH= "C:\\vcpkg\\installed\\x64-windows\\include\\"
+VCPKG_ROCKSDB_LIB_PATH= "C:\\vcpkg\\installed\\x64-windows\\lib\\"
+vcpkgVersion= "2020.06.15"
+vcpkgInstallParamPath= '$(Build.SourcesDirectory)\\.azure-pipelines\\vcpkg'
+vcpkgBinariesPath= 'C:\\Users\\VssAdministrator\\AppData\\Local\\vcpkg\\archives'
+"vcpkg.arch"= 'x64'
diff --git a/setup.cfg b/setup.cfg
@@ -1,7 +1,6 @@
 [metadata]
 name = rocksdb
-version = 0.9.1
-description = Python bindings for RocksDB
+description = Python bindings for RocksDB, primarily for use with Faust
 long_description = file: README.rst
 long_description_content_type = text/x-rst
 url = https://github.com/faust-streaming/python-rocksdb
diff --git a/setup.py b/setup.py
@@ -18,9 +18,6 @@
     '-fno-rtti',
 ]
 
-if platform.system() == 'Darwin':
-    extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++']
-
 if sys.version_info < (3 , 0):
     raise Exception('python-rocksdb requires Python 3.x')
 
@@ -36,10 +33,19 @@
         'libraries': ['rocksdb', 'snappy', 'bz2', 'z', 'lz4'],
     }
 
+if platform.system() == 'Darwin':
+    extra_compile_args += ['-mmacosx-version-min=10.9', '-stdlib=libc++']
+
+if platform.system() == 'Windows':
+    extra_compile_args.remove('-Wextra')
+    extra_compile_args.remove('-Wconversion')
+    ext_args['libraries'].remove('z')
+    ext_args['libraries'].append('zlib')
+
 rocksdb_extension = Extension(
     'rocksdb._rocksdb',
     [
-        'rocksdb/_rocksdb.pyx',
+        os.path.join('rocksdb','_rocksdb.pyx'),
     ],
     extra_compile_args=extra_compile_args,
     language='c++',
@@ -48,14 +54,14 @@
 
 setup(
     name="faust-streaming-rocksdb",
-    version='0.9.1',
-    description="Python bindings for RocksDB",
+    use_scm_version=True,
+    description="Python bindings for RocksDB, primarily for use with Faust",
     keywords='rocksdb',
-    author='Ming Hsuan Tu',
-    author_email="qrnnis2623891@gmail.com",
+    author='William Barnhart',
+    author_email="williambbarnhart@gmail.com",
     url="https://github.com/faust-streaming/python-rocksdb",
     license='BSD License',
-    setup_requires=['setuptools>=25', 'Cython>=0.20'],
+    setup_requires=['setuptools>=25', 'Cython>=0.20', 'setuptools_scm'],
     install_requires=['setuptools>=25'],
     package_dir={'rocksdb': 'rocksdb'},
     packages=find_packages('.'),

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM ubuntu:20.04`
	`1`	`+FROM ubuntu:22.04`
`2`	`2`	`ENV SRC /home/tester/src`
`3`	`3`	`ENV DEBIAN_FRONTEND noninteractive`
`4`	`4`