harden getMetadata and related calls from getting non-utf-8 'json' (#140)

* harden getMetadata and related calls from getting non-utf-8 'json' from PDAL

* whoops

* lern2type

* py.test no longer always available on conda-forge packages

* editable install

* arg order

* no editable

* explicit pytest install

* ditch 3.7 and 3.8

* pip install --upgrade --force-reinstall pytest

* shut off embed for now

* test fixes

* turn on embed tests

* update pypa/gh-action-pypi-publish@release/v1

Author: hobu

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11']
 
     steps:
     - name: Check out python-pdal
@@ -50,6 +50,9 @@ jobs:
       working-directory: ./plugins
       run: pip install .
 
+    - name: Install pytest
+      run: pip install --upgrade --force-reinstall pytest
+
     - name: Test
       run: |
         export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))")
@@ -87,7 +90,7 @@ jobs:
         tree ./dist
 
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       if: github.event_name == 'release' && github.event.action == 'published'
       with:
         user: __token__

pdal/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.2.0"
+__version__ = "3.2.1"
 __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"]
 
 from . import libpdalpython

pdal/libpdalpython.cpp

@@ -134,8 +134,9 @@ namespace pdal {
             MetadataNode root = (StreamableExecutor::getMetadata()).clone("metadata");
             pdal::Utils::toJSON(root, strm);
 
-            py::str pystring(strm.str());
-            pystring.attr("strip");
+
+            py::bytes pybytes(strm.str());
+            py::str pystring ( pybytes.attr("decode")("utf-8", "ignore"));
 
             py::object j;
             j = json.attr("loads")(pystring);
@@ -182,7 +183,9 @@ namespace pdal {
             py::gil_scoped_acquire acquire;
             py::object json = py::module_::import("json");
 
-            py::str pystring(getExecutor()->getQuickInfo());
+            py::bytes pybytes(getExecutor()->getQuickInfo());
+
+            py::str pystring ( pybytes.attr("decode")("utf-8", "ignore"));
             pystring.attr("strip");
 
             py::object j;
@@ -193,7 +196,16 @@ namespace pdal {
         }
 
         py::object getMetadata() {
-            return py::module_::import("json").attr("loads")(getExecutor()->getMetadata());
+            py::object json = py::module_::import("json");
+
+            py::bytes pybytes(getExecutor()->getMetadata());
+            py::str pystring ( pybytes.attr("decode")("utf-8", "ignore"));
+
+            py::object j;
+            j = json.attr("loads")(pystring);
+
+            return j;
+
         }
 
         py::object getSchema() {

test/test_pipeline.py

@@ -34,7 +34,7 @@ def get_pipeline(filename):
 def test_dimensions():
     """Ask PDAL for its valid dimensions list"""
     dims = pdal.dimensions
-    assert 71 < len(dims) < 120
+    assert 71 < len(dims) < 122
 
 
 class TestPipeline:
@@ -500,6 +500,10 @@ def test_fetch(self):
         df = p.get_dataframe(0)
         assert df.size == 17040
 
+    @pytest.mark.skipif(
+        not pdal.pipeline.DataFrame,
+        reason="pandas is not available",
+    )
     def test_load(self):
         r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las"))
         p = r.pipeline()