Set Numpy floor to 1.22 and NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION (#167)

* Set Numpy floor to 1.22 and NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION

* set PY_ARRAY_UNIQUE_SYMBOL to  PDAL_ARRAY_API and only import it in extension initialization. Guard against using the extension against any PDAL verisions < 2.6

* debug PDAL_DRIVER_PATH issues

* bug in --print-plugin-path

* only one os.path.sep

* concat PDAL_DRIVER_PATH with PDAL_PLUGIN_PATH

Author: hobu

.github/workflows/build.yml

@@ -57,8 +57,12 @@ jobs:
 
     - name: Test
       run: |
-        export PDAL_DRIVER_PATH=$(python -m pdal --pdal-plugin-path)
-        echo $PDAL_DRIVER_PATH
+        export PDAL_DRIVER_PATH=$(python -m pdal --pdal-driver-path)
+        export PDAL_PLUGIN_PATH=$(python -m pdal --pdal-plugin-path)
+        echo "PDAL_DRIVER_PATH $PDAL_DRIVER_PATH"
+        echo "PDAL_PLUGIN_PATH $PDAL_PLUGIN_PATH"
+        export PDAL_DRIVER_PATH=$PDAL_PLUGIN_PATH:$PDAL_DRIVER_PATH
+        python -m pdal
         pdal --drivers --debug
         py.test -v test/
 

CMakeLists.txt

@@ -1,5 +1,7 @@
 cmake_minimum_required(VERSION 3.11.0)
-project(pdal-python VERSION)
+project(pdal-python VERSION ${SKBUILD_PROJECT_VERSION}
+                    DESCRIPTION "PDAL Python bindings"
+                    HOMEPAGE_URL "https://github.com/PDAL/Python")
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

pyproject.toml

@@ -26,7 +26,7 @@ classifiers = [
 ]
 
 dependencies = [
-  "numpy"
+  "numpy >= 1.22"
 ]
 
 dynamic = ["version"]
@@ -48,7 +48,7 @@ repository = "https://github.com/PDAL/Python"
 changelog = "https://github.com/PDAL/python/blob/main/README.rst"
 
 [build-system]
-requires = ["scikit-build-core >= 0.9", "numpy",  "pybind11[global]"]
+requires = ["scikit-build-core >= 0.9", "numpy >= 1.22",  "pybind11[global]"]
 build-backend = "scikit_build_core.build"
 
 

src/pdal/PyArray.cpp

@@ -92,9 +92,6 @@ std::string toString(PyObject *pname)
 
 Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true)
 {
-    if (_import_array() < 0)
-        throw pdal_error("Could not import numpy.core.multiarray.");
-
     Py_XINCREF(array);
 
     PyArray_Descr *dtype = PyArray_DTYPE(m_array);

src/pdal/PyArray.hpp

@@ -35,8 +35,17 @@
 #pragma once
 
 #include <pdal/PointView.hpp>
+
+#define NPY_TARGET_VERSION NPY_1_22_API_VERSION
+#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
+
+#define NO_IMPORT_ARRAY
+#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
+
 #include <pdal/io/MemoryViewReader.hpp>
+
 #include <numpy/ndarraytypes.h>
+#include <numpy/arrayobject.h>
 
 #include <vector>
 #include <memory>

src/pdal/PyPipeline.cpp

@@ -211,16 +211,7 @@ std::string PipelineExecutor::getQuickInfo() const
 
 void PipelineExecutor::addArrayReaders(std::vector<std::shared_ptr<Array>> arrays)
 {
-    // Make the symbols in pdal_base global so that they're accessible
-    // to PDAL plugins.  Python dlopen's this extension with RTLD_LOCAL,
-    // which means that without this, symbols in libpdal_base aren't available
-    // for resolution of symbols on future runtime linking.  This is an issue
-    // on Alpine and other Linux variants that don't use UNIQUE symbols
-    // for C++ template statics only.  Without this, you end up with multiple
-    // copies of template statics.
-#ifndef _WIN32
-    ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL);
-#endif
+
     if (arrays.empty())
         return;
 
@@ -320,8 +311,6 @@ PyObject* buildNumpyDescriptor(PointLayoutPtr layout)
 
 PyArrayObject* viewToNumpyArray(PointViewPtr view)
 {
-    if (_import_array() < 0)
-        throw pdal_error("Could not import numpy.core.multiarray.");
 
     PyObject* dtype_dict = buildNumpyDescriptor(view->layout());
     PyArray_Descr *dtype = nullptr;
@@ -344,9 +333,6 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view)
 
 PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh)
 {
-    if (_import_array() < 0)
-        throw pdal_error("Could not import numpy.core.multiarray.");
-
     // Build up a numpy dtype dictionary
     //
     // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1',

src/pdal/PyPipeline.hpp

@@ -35,6 +35,13 @@
 #pragma once
 
 #include <pdal/PipelineManager.hpp>
+
+#define NPY_TARGET_VERSION NPY_1_22_API_VERSION
+#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
+
+#define NO_IMPORT_ARRAY
+#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
+
 #include <numpy/arrayobject.h>
 
 namespace pdal

src/pdal/StreamableExecutor.cpp

@@ -35,6 +35,9 @@
 #include "PyPipeline.hpp"
 #include "StreamableExecutor.hpp"
 
+#define NO_IMPORT_ARRAY
+#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
+
 #include <Python.h>
 #include <numpy/arrayobject.h>
 
@@ -67,8 +70,7 @@ void PythonPointTable::finalize()
 
     // create dtype
     auto gil = PyGILState_Ensure();
-    if (_import_array() < 0)
-        std::cerr << "Could not import array!\n";
+
     PyObject *dtype_dict = buildNumpyDescriptor(&m_layout);
     if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL)
         throw pdal_error("Unable to create numpy dtype");
@@ -102,8 +104,8 @@ void PythonPointTable::py_resizeArray(point_count_t np)
         {
             if (src_idx != dest_idx)
             {
-                PyObject* src_item = PyArray_GETITEM(m_curArray, PyArray_GETPTR1(m_curArray, src_idx));
-                PyArray_SETITEM(m_curArray, PyArray_GETPTR1(m_curArray, dest_idx), src_item);
+                PyObject* src_item = PyArray_GETITEM(m_curArray, (const char*) PyArray_GETPTR1(m_curArray, src_idx));
+                PyArray_SETITEM(m_curArray, (char*) PyArray_GETPTR1(m_curArray, dest_idx), src_item);
                 Py_XDECREF(src_item);
             }
             dest_idx++;

src/pdal/__main__.py

@@ -23,7 +23,7 @@ def print_driver_path(args):
     print (os.environ['PDAL_DRIVER_PATH'])
 
 def print_plugin_path(args):
-    purelib = sysconfig.get_paths()["purelib"]+os.path.sep+"pdal"
+    purelib = sysconfig.get_paths()["purelib"]
 
     if sys.platform == "linux" or sys.platform == "linux2":
         suffix = 'so'

src/pdal/libpdalpython.cpp

@@ -7,6 +7,13 @@
 #include <pdal/pdal_config.hpp>
 #include <pdal/StageFactory.hpp>
 
+#define NPY_TARGET_VERSION NPY_1_22_API_VERSION
+#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
+
+#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
+
+#include <numpy/arrayobject.h>
+
 #include "PyArray.hpp"
 #include "PyDimension.hpp"
 #include "PyPipeline.hpp"
@@ -283,8 +290,12 @@ namespace pdal {
         int _loglevel;
     };
 
+
+
     PYBIND11_MODULE(libpdalpython, m)
     {
+        _import_array();
+
     py::class_<PipelineIterator>(m, "PipelineIterator")
         .def("__iter__", [](PipelineIterator &it) -> PipelineIterator& { return it; })
         .def("__next__", &PipelineIterator::executeNext)
@@ -294,6 +305,7 @@ namespace pdal {
         .def_property_readonly("pipeline", &PipelineIterator::getPipeline)
         .def_property_readonly("metadata", &PipelineIterator::getMetadata);
 
+
     py::class_<Pipeline>(m, "Pipeline")
         .def(py::init<>())
         .def("execute", &Pipeline::execute)
@@ -319,6 +331,12 @@ namespace pdal {
     m.def("getDimensions", &getDimensions);
     m.def("infer_reader_driver", &getReaderDriver);
     m.def("infer_writer_driver", &getWriterDriver);
+
+    if (pdal::Config::versionMajor() < 2)
+        throw pybind11::import_error("PDAL version must be >= 2.6");
+
+    if (pdal::Config::versionMajor() == 2 && pdal::Config::versionMinor() < 6)
+        throw pybind11::import_error("PDAL version must be >= 2.6");
     };
 
 }; // namespace pdal