implement #129, add pandas DataFrame i/o for convenience (#130)

Author: hobu

.github/environment.yml

@@ -8,3 +8,4 @@ dependencies:
   - pdal
   - pytest
   - meshio
+  - pandas

pdal/pipeline.py

@@ -11,6 +11,11 @@
 except ModuleNotFoundError:  # pragma: no cover
     Mesh = None
 
+try:
+    from pandas import DataFrame
+except ModuleNotFoundError:  # pragma: no cover
+    DataFrame = None
+
 from . import drivers, libpdalpython
 
 LogLevelToPDAL = {
@@ -28,13 +33,19 @@ def __init__(
         spec: Union[None, str, Sequence[Stage]] = None,
         arrays: Sequence[np.ndarray] = (),
         loglevel: int = logging.ERROR,
-        json: Optional[str] = None
+        json: Optional[str] = None,
+        dataframes: Sequence[DataFrame] = (),
     ):
+
         if json:
             if spec and json:
                 raise ValueError("provide 'spec' or 'json' arguments, not both")
             spec = json
 
+        # Convert our data frames to Numpy Structured Arrays
+        if dataframes:
+            arrays = [df.to_records() for df in dataframes]
+
         super().__init__()
         self._stages: List[Stage] = []
         if spec:
@@ -112,6 +123,13 @@ def get_meshio(self, idx: int) -> Optional[Mesh]:
             [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))],
         )
 
+
+    def get_dataframe(self, idx: int) -> Optional[DataFrame]:
+        if DataFrame is None:
+            raise RuntimeError("Pandas support requires Pandas to be installed")
+
+        return DataFrame(self.arrays[idx])
+
     def _get_json(self) -> str:
         return self.toJSON()
 

test/test_pipeline.py

@@ -344,7 +344,7 @@ def test_logging_filters_python(self, filename):
         reason="filters.python PDAL plugin is not available",
     )
     def test_filters_python(self):
-        r = pdal.Reader("test/data/autzen-utm.las")
+        r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las"))
         f = pdal.Filter.python(script=__file__, function="a_filter", module="anything")
         count = (r | f).execute()
         assert count == 1065
@@ -364,14 +364,14 @@ def test_only_readers(self):
         np.testing.assert_array_equal(np.concatenate([array1, array1]), array2)
 
     def test_quickinfo(self):
-        r = pdal.Reader("test/data/autzen-utm.las")
+        r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las"))
         p = r.pipeline()
         info = p.quickinfo
         assert 'readers.las' in info.keys()
         assert info['readers.las']['num_points'] == 1065
 
     def test_jsonkwarg(self):
-        pipeline = pdal.Reader("test/data/autzen-utm.las").pipeline().toJSON()
+        pipeline = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")).pipeline().toJSON()
         r = pdal.Pipeline(json=pipeline)
         p = r.pipeline
         assert 'readers.las' in p
@@ -487,6 +487,45 @@ def test_meshio(self, filename):
         assert len(triangles) == 134
         assert triangles[0][0] == 29
 
+class TestDataFrame:
+
+    @pytest.mark.skipif(
+        not pdal.pipeline.DataFrame,
+        reason="pandas is not available",
+    )
+    def test_fetch(self):
+        r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las"))
+        p = r.pipeline()
+        p.execute()
+        df = p.get_dataframe(0)
+        assert df.size == 17040
+
+    def test_load(self):
+        r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las"))
+        p = r.pipeline()
+        p.execute()
+        data = p.arrays[0]
+        df = pdal.pipeline.DataFrame
+        dataframes = [df(data), df(data), df(data)]
+        filter_intensity = """{
+          "pipeline":[
+            {
+              "type":"filters.range",
+              "limits":"Intensity[100:300)"
+            }
+          ]
+        }"""
+        p = pdal.Pipeline(filter_intensity, dataframes = dataframes)
+        p.execute()
+        arrays = p.arrays
+        assert len(arrays) == 3
+
+        # We copied the array three times. Sum the Intensity values
+        # post filtering to see if we had our intended effect
+        for data in arrays:
+            assert len(data) == 387
+            assert data["Intensity"].sum() == 57684
+
 
 class TestPipelineIterator:
     @pytest.mark.parametrize("filename", ["sort.json", "sort.py"])