fix: pybind11 update, support PyPy / Python 3.9 / cleanup (#443)

henryiii · web-flow · commit defaee77094a · 2020-10-21T13:53:04.000-04:00
* feat: pybind11 2.6 dev

* chore: update to newer pybind11

* chore: use vector void impl

* fix: update requirements

* docs: update CHANGELOG

* chore: missing constructor now present

* chore: update pybind11 to 2.6.0rc1, use get_pointer()

cibuildwheel too

* ci: bump to pybind11 2.6.0 rc 2

* chore: another pybind11 update, should fix pypy3 on windows

* ci(wheels): use latest MSVC on PyPy

* fix: tighten strides constructor for PyPy on Windows

* chore: update to 2.6.0
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -35,7 +35,7 @@ jobs:
       run: apt-get update && apt-get install -y python3-dev python3-pip
 
     - name: Install extra requirements
-      run: python3 -m pip install setuptools_scm
+      run: python3 -m pip install setuptools_scm toml
 
     - name: Configure
       run: cmake -S . -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--warnings-as-errors=*"
@@ -49,7 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["2.7", "3.5", "3.8"] # Waiting on CPython or pybind11 fix: , "3.9-dev"]
+        python-version: ["2.7", "3.5", "3.8", "3.9", "pypy3"]
 
     name: CMake Python ${{ matrix.python-version }}
 
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -18,7 +18,8 @@ on:
 
 env:
   CIBW_TEST_EXTRAS: test
-  CIBW_TEST_COMMAND: pytest {project}/tests
+  # Disabled currently - needs NumPy 3.9 wheels
+  CIBW_TEST_COMMAND_: "" # pytest {project}/tests
   SETUPTOOLS_SCM_PRETEND_VERSION: ${{ github.event.inputs.overrideVersion }}
 
 
@@ -34,10 +35,10 @@ jobs:
     - uses: actions/setup-python@v2
 
     - name: Install deps
-      run: python -m pip install "setuptools>=42" "setuptools_scm[toml]>=4.1.0" twine
+      run: python -m pip install build twine
 
     - name: Build SDist
-      run: python setup.py sdist
+      run: python -m build -s
 
     - name: Check metadata
       run: twine check dist/*
@@ -51,12 +52,14 @@ jobs:
     name: ${{ matrix.type }} wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-18.04, windows-latest, macos-latest]
         type: [Standard]
 
         include:
-          - type: Standard
+          - os: ubuntu-18.04
+            type: Standard
             CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010
             CIBW_MANYLINUX_I686_IMAGE: manylinux2010
 
@@ -73,12 +76,12 @@ jobs:
     - uses: actions/setup-python@v2
 
     - name: Install cibuildwheel
-      run: python -m pip install cibuildwheel==1.5.5
+      run: python -m pip install cibuildwheel==1.6.3
 
     - name: Build wheel
       run: python -m cibuildwheel --output-dir wheelhouse
       env:
-        CIBW_SKIP: pp* cp27-win*
+        CIBW_SKIP: cp27-win* pp27-win32
         CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.CIBW_MANYLINUX_I686_IMAGE }}
         CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.CIBW_MANYLINUX_X86_64_IMAGE }}
 
@@ -109,7 +112,7 @@ jobs:
     - uses: actions/setup-python@v2
 
     - name: Install cibuildwheel
-      run: python -m pip install cibuildwheel==1.5.5
+      run: python -m pip install cibuildwheel==1.6.3
 
     - uses: ilammy/msvc-dev-cmd@v1
 
@@ -127,7 +130,7 @@ jobs:
     - name: Build 32-bit wheel
       run: python -m cibuildwheel --output-dir wheelhouse
       env:
-        CIBW_BUILD: cp27-win32
+        CIBW_BUILD: cp27-win32 pp27-win32
         DISTUTILS_USE_SDK: 1
         MSSdk: 1
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -174,27 +174,6 @@ install(TARGETS _core DESTINATION "boost_histogram")
 # Tests (Requires pytest to be available to run)
 include(CTest)
 
-# Check to see if modules are importable.
-function(python_import)
-  foreach(PACKAGE_NAME IN LISTS ARGN)
-    string(REPLACE "-" "_" PYTHON_PACKAGE_NAME "${PACKAGE_NAME}")
-    execute_process(
-      COMMAND ${PYTHON_EXECUTABLE} -c "import ${PYTHON_PACKAGE_NAME}"
-      RESULT_VARIABLE RESULT_PRESENT
-      OUTPUT_QUIET ERROR_QUIET)
-    if(RESULT_PRESENT)
-      list(APPEND FAILED_NAMES "${PACKAGE_NAME}")
-    endif()
-  endforeach()
-  if(FAILED_NAMES)
-    list(JOIN FAILED_NAMES " " FAILED_NAMES)
-    message(
-      WARNING
-        "Dependencies missing, try installing:\n  ${PYTHON_EXECUTABLE} -m pip install ${FAILED_NAMES}\n"
-    )
-  endif()
-endfunction()
-
 if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/boost_histogram/version.py")
   set(VERSION_REGEX [=[version[ \t]*=[ \t]*["']([0-9]+\.[0-9]+\.[0-9]+)]=])
 
@@ -205,7 +184,8 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/boost_histogram/version.py")
   # Pick out just the version
   string(REGEX MATCH [=[[0-9]+\.[0-9]+\.[0-9]+]=] VERSION_STRING "${VERSION_STRING}")
 else()
-  python_import(setuptools_scm toml)
+  pybind11_find_import(setuptools_scm REQUIRED)
+  pybind11_find_import(toml REQUIRED)
 
   execute_process(
     COMMAND
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -85,7 +85,6 @@ source ./env_cmake/bin/activate
 pip install -r dev-requirements.txt
 cmake -S . -B build-debug \
     -GNinja \
-    -DPYTHON_EXECUTABLE=$(which python) \
     -DCMAKE_INSTALL_PREFIX=$(python -c "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(plat_specific=False,standard_lib=False))")
 cmake --build build-debug -j4
 cmake --install build-debug # Option 3 only
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -14,6 +14,7 @@ graft extern/variant2/include
 
 graft extern/pybind11/include
 graft extern/pybind11/tools
+graft extern/pybind11/pybind11
 include extern/pybind11/CMakeLists.txt
 
 prune docs/_build
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,5 +1,5 @@
 numpy
-pytest <6.1
+pytest !=6.1.0
 pytest-benchmark
 setuptools >=42
 setuptools_scm[toml] >=3.4,!=4.0.0
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
 # What's new in boost-histogram
 
 ## Version 0.11
+
+### Version 0.11.1
+
+Updating pybind11 to 2.6.0. Features:
+
+* Python 3.9 support
+* PyPy support
+* Warnings on latest AppleClang fixed
+* 40% faster accumulator fills, simpler implementation
+* Segfaults when passing an object with a throwing repr fixed
+* kwargs replaced older workarounds (partially at the moment)
+* Using new `py::type` instead of `pybind11::detail` usage
+* Enhanced CMake support, finds conda and venv now, uses `pybind11_find_import`
+* Using setuptools support from pybind11 (previously vendored, so benefits have been available since 0.11.0)
+
 ### Version 0.11.0
 
 A release focused on preparing for the upcoming Hist 2.0 release.
diff --git a/extern/pybind11 b/extern/pybind11
@@ -1 +1 @@
-Subproject commit 3b1dbebabc801c9cf6f0953a4c20b904d444f879
+Subproject commit 59a2ac2745d8a57ac94c6accced73620d59fb844
diff --git a/include/bh_python/array_like.hpp b/include/bh_python/array_like.hpp
@@ -23,13 +23,13 @@ py::array_t<T> array_like(py::object obj) {
         return py::array_t<T>(shape);
     }
     auto arr = py::cast<py::array>(obj);
-    std::vector<ssize_t> strides;
+    std::vector<py::ssize_t> strides;
     strides.reserve(static_cast<std::size_t>(arr.ndim()));
     for(int i = 0; i < arr.ndim(); ++i) {
         strides.emplace_back(arr.strides()[i] / arr.itemsize()
-                             * static_cast<ssize_t>(sizeof(T)));
+                             * static_cast<py::ssize_t>(sizeof(T)));
     }
-    return py::array_t<T>(bh::detail::span<const ssize_t>(
-                              arr.shape(), static_cast<std::size_t>(arr.ndim())),
-                          strides);
+    return py::array_t<T>{bh::detail::span<const py::ssize_t>{
+                              arr.shape(), static_cast<std::size_t>(arr.ndim())},
+                          strides};
 }
diff --git a/include/bh_python/axis.hpp b/include/bh_python/axis.hpp
@@ -214,7 +214,7 @@ py::array_t<double> edges(const A& ax, bool flow = false, bool numpy_upper = fal
             = flow && (bh::axis::traits::get_options<AX>::test(option::overflow));
 
         py::array_t<double> edges(
-            static_cast<std::size_t>(ax.size() + 1 + overflow + underflow));
+            static_cast<py::ssize_t>(ax.size() + 1 + overflow + underflow));
 
         for(index_type i = -underflow; i <= ax.size() + overflow; ++i)
             edges.mutable_at(i + underflow) = ax.value(i);
@@ -238,7 +238,7 @@ py::array_t<double> edges(const A& ax, bool flow = false, bool numpy_upper = fal
                 = flow && bh::axis::traits::get_options<AX>::test(option::overflow);
 
             py::array_t<double> edges(
-                static_cast<std::size_t>(ax.size() + 1 + overflow));
+                static_cast<py::ssize_t>(ax.size() + 1 + overflow));
 
             for(bh::axis::index_type i = 0; i <= ax.size() + overflow; ++i)
                 edges.mutable_at(i) = i;
@@ -251,15 +251,15 @@ py::array_t<double> edges(const A& ax, bool flow = false, bool numpy_upper = fal
 
 template <class A>
 py::array_t<double> centers(const A& ax) {
-    py::array_t<double> result(static_cast<std::size_t>(ax.size()));
+    py::array_t<double> result(ax.size());
     for(bh::axis::index_type i = 0; i < ax.size(); ++i)
         result.mutable_data()[i] = unchecked_center(ax, i);
     return result;
 }
 
 template <class A>
 py::array_t<double> widths(const A& ax) {
-    py::array_t<double> result(static_cast<std::size_t>(ax.size()));
+    py::array_t<double> result(ax.size());
     bh::detail::static_if<bh::axis::traits::is_continuous<A>>(
         [](py::array_t<double>& result, const auto& ax) {
             std::transform(ax.begin(),
diff --git a/include/bh_python/make_pickle.hpp b/include/bh_python/make_pickle.hpp
@@ -197,7 +197,7 @@ class tuple_oarchive {
     std::enable_if_t<std::is_arithmetic<T>::value == true, tuple_oarchive&>
     operator<<(const std::vector<T>& v) {
         // fast version for vector of arithmetic types
-        py::array_t<T> a(v.size(), v.data());
+        py::array_t<T> a(static_cast<py::ssize_t>(v.size()), v.data());
         this->operator<<(static_cast<const py::object&>(a));
         return *this;
     }
@@ -216,7 +216,7 @@ class tuple_oarchive {
     std::enable_if_t<std::is_arithmetic<T>::value == true, tuple_oarchive&>
     operator<<(const bh::detail::array_wrapper<T>& w) {
         // fast version
-        py::array_t<T> a(w.size, w.ptr);
+        py::array_t<T> a(static_cast<py::ssize_t>(w.size), w.ptr);
         this->operator<<(static_cast<const py::object&>(a));
         return *this;
     }
diff --git a/include/bh_python/register_histogram.hpp b/include/bh_python/register_histogram.hpp
@@ -80,9 +80,7 @@ auto register_histogram(py::module& m, const char* name, const char* desc) {
         .def_property_readonly_static(
             "_storage_type",
             [](py::object) {
-                return py::detail::get_type_handle(
-                    typeid(typename histogram_t::storage_type), true);
-                // Change to py::type<T>() if added to pybind11
+                return py::type::of<typename histogram_t::storage_type>();
             })
 
         ;
@@ -123,12 +121,8 @@ auto register_histogram(py::module& m, const char* name, const char* desc) {
         .def(
             "view",
             [](py::object self, bool flow) {
-                auto& h   = py::cast<histogram_t&>(self);
-                auto info = make_buffer(h, flow);
-                return py::array(
-                    pybind11::dtype(info), info.shape, info.strides, info.ptr, self);
-                // Note that, due to the missing signature py::array(info, self), we
-                // have to write this out fully here. TODO: Make PR to pybind11
+                auto& h = py::cast<histogram_t&>(self);
+                return py::array(make_buffer(h, flow), self);
             },
             "flow"_a = false)
 
diff --git a/include/bh_python/storage.hpp b/include/bh_python/storage.hpp
@@ -80,7 +80,7 @@ template <class Archive>
 void save(Archive& ar, const storage::atomic_int64& s, unsigned /* version */) {
     // We cannot view the memory as a numpy array, because the internal layout of
     // std::atomic is undefined. So no reinterpret_casts are allowed.
-    py::array_t<std::int64_t> a(s.size());
+    py::array_t<std::int64_t> a(static_cast<py::ssize_t>(s.size()));
     std::copy(s.begin(), s.end(), a.mutable_data());
     ar << a;
 }
@@ -105,7 +105,8 @@ void save(Archive& ar,
                       && sizeof(T) == 2 * sizeof(double),
                   "weighted_sum cannot be fast serialized");
     // view storage buffer as flat numpy array
-    py::array_t<double> a(s.size() * 2, reinterpret_cast<const double*>(s.data()));
+    py::array_t<double> a(static_cast<py::ssize_t>(s.size()) * 2,
+                          reinterpret_cast<const double*>(s.data()));
     ar << a;
 }
 
@@ -131,7 +132,8 @@ void save(Archive& ar,
                       && sizeof(T) == 3 * sizeof(double),
                   "mean cannot be fast serialized");
     // view storage buffer as flat numpy array
-    py::array_t<double> a(s.size() * 3, reinterpret_cast<const double*>(s.data()));
+    py::array_t<double> a(static_cast<py::ssize_t>(s.size()) * 3,
+                          reinterpret_cast<const double*>(s.data()));
     ar << a;
 }
 
@@ -157,7 +159,8 @@ void save(Archive& ar,
                       && sizeof(T) == 4 * sizeof(double),
                   "weighted_mean cannot be fast serialized");
     // view storage buffer as flat numpy array
-    py::array_t<double> a(s.size() * 4, reinterpret_cast<const double*>(s.data()));
+    py::array_t<double> a(static_cast<py::ssize_t>(s.size()) * 4,
+                          reinterpret_cast<const double*>(s.data()));
     ar << a;
 }
 
diff --git a/include/bh_python/transform.hpp b/include/bh_python/transform.hpp
@@ -71,8 +71,7 @@ struct func_transform {
             auto c = py::reinterpret_borrow<py::capsule>(
                 PyCFunction_GET_SELF(cfunc.ptr()));
 
-            // This calls operator T* (PyCapsule_SetPointer)
-            auto rec = static_cast<py::detail::function_record*>(c);
+            auto rec = c.get_pointer<py::detail::function_record>();
 
             if(rec && rec->is_stateless
                && py::detail::same_type(
diff --git a/setup.cfg b/setup.cfg
@@ -50,11 +50,12 @@ exclude =
 
 [tool:pytest]
 junit_family=xunit2
-addopts = --benchmark-disable -Wd
+addopts = --benchmark-disable -Wd --strict-markers
 xfail_strict = True
 testpaths =
     tests
-
+required_plugins =
+    pytest-benchmark
 
 [flake8]
 max-line-length = 80
@@ -102,6 +103,8 @@ ignore =
   docs/**
   extern/**/*.py
   extern/**/*.md
+  extern/**/*.rst
+  extern/**/*.toml
   extern/*/checks/**
   extern/*/test/**
   extern/*/benchmark/**
diff --git a/setup.py b/setup.py
@@ -6,11 +6,12 @@
 import sys
 import os
 
-sys.path.append(os.path.abspath(os.path.dirname(__file__)))
-from setup_helpers import Pybind11Extension, ParallelCompile  # noqa: E402
+DIR = os.path.abspath(os.path.dirname(__file__))
 
-del sys.path[-1]
+sys.path.append(os.path.join(DIR, "extern", "pybind11"))
+from pybind11.setup_helpers import Pybind11Extension, ParallelCompile  # noqa: E402
 
+del sys.path[-1]
 
 # Use the environment variable NPY_NUM_BUILD_JOBS
 ParallelCompile("NPY_NUM_BUILD_JOBS").install()
@@ -32,7 +33,6 @@
     "extern/core/include",
     "extern/histogram/include",
     "extern/mp11/include",
-    "extern/pybind11/include",
     "extern/throw_exception/include",
     "extern/variant2/include",
 ]
@@ -43,17 +43,17 @@
         SRC_FILES,
         include_dirs=INCLUDE_DIRS,
         cxx_std=14,
-        include_pybind11=False,
+        extra_compile_args=["/d2FH4-"] if sys.platform.startswith("win32") else [],
     )
 ]
 
 
 extras = {
     "test": ["pytest", "pytest-benchmark"],
     "docs": [
-        "Sphinx>=2.0.0",
+        "Sphinx~=3.0",
         "recommonmark>=0.5.0",
-        "sphinx_book_theme==0.30.0",
+        "sphinx_book_theme==0.38.0",
         "nbsphinx",
         "sphinx_copybutton",
     ],
diff --git a/setup_helpers.py b/setup_helpers.py
diff --git a/src/boost_histogram/_internal/hist.py b/src/boost_histogram/_internal/hist.py
diff --git a/src/register_accumulators.cpp b/src/register_accumulators.cpp
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_histogram.py b/tests/test_histogram.py
diff --git a/tests/test_internal_signatures.py b/tests/test_internal_signatures.py
diff --git a/tests/test_pickle.py b/tests/test_pickle.py

Original file line number	Diff line number	Diff line change
`@@ -23,13 +23,13 @@ py::array_t<T> array_like(py::object obj) {`
`23`	`23`	`return py::array_t<T>(shape);`
`24`	`24`	`}`
`25`	`25`	`auto arr = py::cast<py::array>(obj);`
`26`		`- std::vector<ssize_t> strides;`
	`26`	`+ std::vector<py::ssize_t> strides;`
`27`	`27`	`strides.reserve(static_cast<std::size_t>(arr.ndim()));`
`28`	`28`	`for(int i = 0; i < arr.ndim(); ++i) {`
`29`	`29`	`strides.emplace_back(arr.strides()[i] / arr.itemsize()`
`30`		`- * static_cast<ssize_t>(sizeof(T)));`
	`30`	`+ * static_cast<py::ssize_t>(sizeof(T)));`
`31`	`31`	`}`
`32`		`- return py::array_t<T>(bh::detail::span<const ssize_t>(`
`33`		`- arr.shape(), static_cast<std::size_t>(arr.ndim())),`
`34`		`- strides);`
	`32`	`+ return py::array_t<T>{bh::detail::span<const py::ssize_t>{`
	`33`	`+ arr.shape(), static_cast<std::size_t>(arr.ndim())},`
	`34`	`+ strides};`
`35`	`35`	`}`