Refactor PDAL Python support based on scikit-build and remove it from main PDAL library

Author: hobu

.gitignore

@@ -1,5 +1,7 @@
 pdal/libpdalpython.cpp
 *.pyc
+_skbuild/*
+.vscode/*
 __pycache__
 build/*
 PDAL.egg-info/*

README.rst

@@ -2,17 +2,39 @@
 PDAL
 ================================================================================
 
-The PDAL Python extension allows you to process data with PDAL into `Numpy`_
-arrays. Additionally, you can use it to fetch `schema`_ and `metadata`_ from
+PDAL Python support allows you to process data with PDAL into `Numpy`_
+arrays. It supports embedding Python in PDAL pipelines with the `readers.numpy <https://pdal.io/stages/readers.numpy.html>`__
+and `filters.python <https://pdal.io/stages/filters.python.html>`__ stages, and it provides a PDAL extension module to control
+Python interaction with PDAL.
+
+Additionally, you can use it to fetch `schema`_ and `metadata`_ from
 PDAL operations.
 
+Installation
+--------------------------------------------------------------------------------
+
+PyPI
+................................................................................
+
+PDAL Python support is installable via PyPI:
+
+.. code-block::
+
+    pip install PDAL
+
+GitHub
+................................................................................
+
 The repository for PDAL's Python extension is available at https://github.com/PDAL/python
 
-It is released independently from PDAL itself as of PDAL 1.7.
+Python support released independently from PDAL itself as of PDAL 1.7.
 
 Usage
 --------------------------------------------------------------------------------
 
+Simple
+................................................................................
+
 Given the following pipeline, which simply reads an `ASPRS LAS`_ file and
 sorts it by the ``X`` dimension:
 
@@ -34,30 +56,107 @@ sorts it by the ``X`` dimension:
 
     import pdal
     pipeline = pdal.Pipeline(json)
-    pipeline.validate() # check if our JSON and options were good
-    pipeline.loglevel = 8 #really noisy
     count = pipeline.execute()
     arrays = pipeline.arrays
     metadata = pipeline.metadata
     log = pipeline.log
 
+Reading  using Numpy Arrays
+................................................................................
+
+The following more complex scenario demonstrates the full cycling between
+PDAL and Python:
+
+ * Read a small testfile from GitHub into a Numpy array
+ * Filters those arrays with Numpy for Intensity
+ * Pass the filtered array to PDAL to be filtered again
+ * Write the filtered array to an LAS file.
+
+.. code-block:: python
+
+    data = "https://github.com/PDAL/PDAL/blob/master/test/data/las/1.2-with-color.las?raw=true"
+
+
+    json = """
+        {
+          "pipeline": [
+            {
+                "type": "readers.las",
+                "filename": "%s"
+            }
+          ]
+        }"""
+
+    import pdal
+    import numpy as np
+    pipeline = pdal.Pipeline(json % data)
+    count = pipeline.execute()
+
+    # get the data from the first array
+    # [array([(637012.24, 849028.31, 431.66, 143, 1, 1, 1, 0, 1,  -9., 132, 7326, 245380.78254963,  68,  77,  88),
+    # dtype=[('X', '<f8'), ('Y', '<f8'), ('Z', '<f8'), ('Intensity', '<u2'), ('ReturnNumber', 'u1'), ('NumberOfReturns', 'u1'), ('ScanDirectionFlag', 'u1'), ('EdgeOfFlightLine', 'u1'), ('Classification', 'u1'), ('ScanAngleRank', '<f4'), ('UserData', 'u1'), ('PointSourceId', '<u2'), ('GpsTime', '<f8'), ('Red', '<u2'), ('Green', '<u2'), ('Blue', '<u2')])
+
+    arr = pipeline.arrays[0]
+    print (len(arr)) # 1065 points
+
+
+    # Filter out entries that have intensity < 50
+    intensity = arr[arr['Intensity'] > 30]
+    print (len(intensity)) # 704 points
+
+
+    # Now use pdal to clamp points that have intensity
+    # 100 <= v < 300, and there are 387
+    clamp =u"""{
+      "pipeline":[
+        {
+          "type":"filters.range",
+          "limits":"Intensity[100:300)"
+        }
+      ]
+    }"""
+
+    p = pdal.Pipeline(clamp, [intensity])
+    count = p.execute()
+    clamped = p.arrays[0]
+    print (count)
+
+    # Write our intensity data to an LAS file
+    output =u"""{
+      "pipeline":[
+        {
+          "type":"writers.las",
+          "filename":"clamped.las",
+          "offset_x":"auto",
+          "offset_y":"auto",
+          "offset_z":"auto",
+          "scale_x":0.01,
+          "scale_y":0.01,
+          "scale_z":0.01
+        }
+      ]
+    }"""
+
+    p = pdal.Pipeline(output, [clamped])
+    count = p.execute()
+    print (count)
+
+
+
 
 .. _`Numpy`: http://www.numpy.org/
 .. _`schema`: http://www.pdal.io/dimensions.html
 .. _`metadata`: http://www.pdal.io/development/metadata.html
 
 
-.. image:: https://travis-ci.org/PDAL/python.svg?branch=master
-    :target: https://travis-ci.org/PDAL/python
-
 .. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d
    :target: https://ci.appveyor.com/project/hobu/python/
 
 Requirements
 ================================================================================
 
-* PDAL 1.7+
-* Python >=2.7 (including Python 3.x)
+* PDAL 2.1+
+* Python >=3.6
 * Cython (eg :code:`pip install cython`)
 * Packaging (eg :code:`pip install packaging`)
 

pdal/CMakeLists.txt

@@ -7,13 +7,13 @@ include(${PROJECT_SOURCE_DIR}/macros.cmake NO_POLICY_SCOPE)
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_BUILD_TYPE "Debug")
+set(CMAKE_BUILD_TYPE "Release")
 
 enable_testing()
 
 # Python-finding settings
 set(Python3_FIND_STRATEGY "LOCATION")
-set(Python3_FIND_REGISTRY "LAST")	  
+set(Python3_FIND_REGISTRY "LAST")
 set(Python3_FIND_FRAMEWORK "LAST")
 find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
 
@@ -33,24 +33,24 @@ execute_process(
     OUTPUT_STRIP_TRAILING_WHITESPACE
 )
 
-execute_process(
-    COMMAND
-    ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('LDSHARED').split(' ', 1)[1])"
-    OUTPUT_VARIABLE PYTHON_LDSHARED
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-
-if (NOT Py_ENABLE_SHARED)
-    message(STATUS "Python ${Python3_EXECUTABLE} is statically linked")
-    if (APPLE)
-        # conda gives us -bundle, which isn't valid
-        message(STATUS "Removing extra -bundle argument from sysconfig.get_config_var('LDSHARED')")
-        string(REPLACE  "-bundle" "" PYTHON_LDSHARED "${PYTHON_LDSHARED}")
-        string(STRIP  ${PYTHON_LDSHARED} PYTHON_LDSHARED)
-    endif()
-  #  set(Python3_LIBRARIES ${PYTHON_LDSHARED})
-    message(STATUS "Setting Python3_LIBRARIES to '${Python3_LIBRARIES}' due to static Python")
-endif()
+#execute_process(
+#    COMMAND
+#    ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('LDSHARED').split(' ', 1)[1])"
+#    OUTPUT_VARIABLE PYTHON_LDSHARED
+#    OUTPUT_STRIP_TRAILING_WHITESPACE
+#)
+
+#if (NOT Py_ENABLE_SHARED)
+#    message(STATUS "Python ${Python3_EXECUTABLE} is statically linked")
+#    if (APPLE)
+#        # conda gives us -bundle, which isn't valid
+#        message(STATUS "Removing extra -bundle argument from sysconfig.get_config_var('LDSHARED')")
+#        string(REPLACE  "-bundle" "" PYTHON_LDSHARED "${PYTHON_LDSHARED}")
+#        string(STRIP  ${PYTHON_LDSHARED} PYTHON_LDSHARED)
+#    endif()
+#  #  set(Python3_LIBRARIES ${PYTHON_LDSHARED})
+#    message(STATUS "Setting Python3_LIBRARIES to '${Python3_LIBRARIES}' due to static Python")
+#endif()
 
 set(EXTENSION_SRC
     PyArray.cpp
@@ -76,15 +76,15 @@ python_extension_module(${extension})
 install(TARGETS ${extension} LIBRARY DESTINATION ${PROJECT_NAME})
 
 PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy
-    FILES  
+    FILES
         ./io/NumpyReader.cpp
         ./io/NumpyReader.hpp
         ./plang/Invocation.cpp
         ./plang/Environment.cpp
         ./plang/Redirector.cpp
         ./plang/Script.cpp
-    LINK_WITH 
-        ${PDAL_LIBRARIES} 
+    LINK_WITH
+        ${PDAL_LIBRARIES}
         ${Python3_LIBRARIES}
         ${CMAKE_DL_LIBS}
     SYSTEM_INCLUDES
@@ -122,15 +122,15 @@ add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src
                  EXCLUDE_FROM_ALL)
 
 PDAL_PYTHON_ADD_PLUGIN(python_filter filter python
-    FILES  
+    FILES
         ./filters/PythonFilter.cpp
         ./filters/PythonFilter.hpp
         ./plang/Invocation.cpp
         ./plang/Environment.cpp
         ./plang/Redirector.cpp
         ./plang/Script.cpp
-    LINK_WITH 
-        ${PDAL_LIBRARIES} 
+    LINK_WITH
+        ${PDAL_LIBRARIES}
         ${Python3_LIBRARIES}
         ${CMAKE_DL_LIBS}
     SYSTEM_INCLUDES
@@ -145,8 +145,12 @@ PDAL_PYTHON_ADD_TEST(pdal_io_numpy_test
     FILES
         ./test/NumpyReaderTest.cpp
         ./test/Support.cpp
+        ./plang/Invocation.cpp
+        ./plang/Environment.cpp
+        ./plang/Redirector.cpp
+        ./plang/Script.cpp
     LINK_WITH
-        ${numpy_reader} 
+        ${numpy_reader}
         ${Python3_LIBRARIES}
         ${PDAL_LIBRARIES}
         ${CMAKE_DL_LIBS}
@@ -160,8 +164,10 @@ PDAL_PYTHON_ADD_TEST(pdal_filters_python_test
     FILES
         ./test/PythonFilterTest.cpp
         ./test/Support.cpp
-                ./plang/Invocation.cpp
+        ./plang/Invocation.cpp
         ./plang/Environment.cpp
+        ./plang/Redirector.cpp
+        ./plang/Script.cpp
     LINK_WITH
         ${python_filter}
         ${Python3_LIBRARIES}

pdal/test/PythonFilterTest.cpp

@@ -38,6 +38,7 @@
 #include <pdal/StageFactory.hpp>
 #include <pdal/io/FauxReader.hpp>
 #include <pdal/filters/StatsFilter.hpp>
+#include <pdal/util/FileUtils.hpp>
 
 #include "../plang/Invocation.hpp"
 #include "../plang/Environment.hpp"
@@ -138,7 +139,7 @@ TEST_F(PythonFilterTest, pipelineJSON)
     PipelineManager manager;
 
     manager.readPipeline(
-        Support::configuredpath("programmable-update-y-dims.json"));
+        Support::datapath("programmable-update-y-dims.json"));
     manager.execute();
     PointViewSet viewSet = manager.views();
     EXPECT_EQ(viewSet.size(), 1u);
@@ -569,7 +570,7 @@ TEST_F(PredicateFilterTest, PredicateFilterTest_PipelineJSON)
 {
     PipelineManager mgr;
 
-    mgr.readPipeline(Support::configuredpath("from-module.json"));
+    mgr.readPipeline(Support::datapath("from-module.json"));
     point_count_t cnt = mgr.execute();
     EXPECT_EQ(cnt, 1u);
 }
@@ -578,7 +579,7 @@ TEST_F(PredicateFilterTest, PredicateFilterTest_EmbedJSON)
 {
     PipelineManager mgr;
 
-    mgr.readPipeline(Support::configuredpath("predicate-embed.json"));
+    mgr.readPipeline(Support::datapath("predicate-embed.json"));
     point_count_t cnt = mgr.execute();
     EXPECT_EQ(cnt, 1u);
 }
@@ -780,6 +781,9 @@ TEST(PLangTest, log)
 {
     // verify we can redirect the stdout inside the python script
 
+    std::string logfile("mylog_three.txt");
+//    std::string logfile(Support::temppath("mylog_three.txt"));
+
     Options reader_opts;
     {
         BOX3D bounds(1.0, 2.0, 3.0, 101.0, 102.0, 103.0);
@@ -791,7 +795,7 @@ TEST(PLangTest, log)
         reader_opts.add(opt2);
         reader_opts.add(opt3);
 
-        Option optlog("log", Support::temppath("mylog_three.txt"));
+        Option optlog("log", logfile);
         reader_opts.add(optlog);
     }
 
@@ -810,7 +814,7 @@ TEST(PLangTest, log)
             );
         const Option module("module", "xModule");
         const Option function("function", "xfunc");
-        xfilter_opts.add("log", Support::temppath("mylog_three.txt"));
+        xfilter_opts.add("log", logfile);
         xfilter_opts.add(source);
         xfilter_opts.add(module);
         xfilter_opts.add(function);
@@ -834,16 +838,15 @@ TEST(PLangTest, log)
         EXPECT_EQ(view->size(), 750u);
     }
 
-    bool ok = Support::compare_text_files(
-        Support::temppath("mylog_three.txt"),
+    bool ok = Support::compare_text_files(logfile,
         Support::datapath("logs/log_py.txt"));
 
     // TODO: fails on Windows
     // unknown file: error: C++ exception with description "pdalboost::filesystem::remove:
     // The process cannot access the file because it is being used by another process:
     // "C:/projects/pdal/test/data/../temp/mylog_three.txt"" thrown in the test body.
-    //if (ok)
-    //    FileUtils::deleteFile(Support::temppath("mylog_three.txt"));
+    if (ok)
+        FileUtils::deleteFile(Support::temppath("mylog_three.txt"));
 
     EXPECT_TRUE(ok);
 }
@@ -1110,7 +1113,7 @@ static void run_pipeline(std::string const& pipeline)
     const std::string cmd = "pdal pipeline";
 
     std::string output;
-    std::string file(Support::configuredpath(pipeline));
+    std::string file(Support::datapath(pipeline));
     int stat = pdal::Utils::run_shell_command(cmd + " " + file, output);
     EXPECT_EQ(0, stat);
     if (stat)

pdal/test/data/1.2-with-color.las

@@ -1,9 +1,9 @@
 {
   "pipeline":[
-    "./1.2-with-color.las",
+    "./pdal/test/data/1.2-with-color.las",
     {
       "type": "filters.python",
-      "script": "./test1.py",
+      "script": "./pdal/test/data/test1.py",
       "module": "anything",
       "function": "fff"
     }

pdal/test/data/autzen-utm.las

@@ -0,0 +1 @@
+Testing log output through python script.