M11.1 v1.9.2 Keep extra-dimensions Distance and Normal (#80)

* Do not clean out dims Normal and Distance

* Keep extra_dims only if they exist in Cleaner.

* Add test for Cleaning with a non-existing dim

* Flake8 fix

* Fix Cleaner test for edge case of empty str for extra-dims

Author: CharlesGaydon

configs/data_format/default.yaml

@@ -2,6 +2,8 @@
 las_dimensions:
   # input
   classification: Classification  # las format
+  terrascan_normal: Normal
+  terrascan_distance: Distance
 
   # Extra dims
   # ATTENTION: If extra dimensions are added, you may want to add them in cleaning.in parameter as well.
@@ -57,12 +59,14 @@ codes:
 
 
 cleaning:
-    # Extra dims that are kept when application starts. Others are removed to lighten the LAS.
+  # Extra dims that are kept when application starts. Others are removed to lighten the LAS.
   input_building:
     _target_: lidar_prod.tasks.cleaning.Cleaner
     extra_dims:
       - "${data_format.las_dimensions.ai_building_proba}=float"
       - "${data_format.las_dimensions.entropy}=float"
+      - "${data_format.las_dimensions.terrascan_normal}=uint"
+      - "${data_format.las_dimensions.terrascan_distance}=uint"
       # - "${data_format.las_dimensions.ai_vegetation_proba}=float"
       # - "${data_format.las_dimensions.ai_unclassified_proba}=float"
   output_building:
@@ -74,9 +78,11 @@ cleaning:
       # - "${data_format.las_dimensions.ai_vegetation_proba}=float"
       - "${data_format.las_dimensions.entropy}=float"
       - "${data_format.las_dimensions.ai_building_identified}=uint"
+      - "${data_format.las_dimensions.terrascan_normal}=uint"
+      - "${data_format.las_dimensions.terrascan_distance}=uint"
       # - "${data_format.las_dimensions.ai_vegetation_unclassified_groups}=uint"
   input_vegetation_unclassified:
-  # Extra dims added for storing the result of the vegetation/unclassified detection.
+    # Extra dims added for storing the result of the vegetation/unclassified detection.
     _target_: lidar_prod.tasks.cleaning.Cleaner
     extra_dims:
       - "${data_format.las_dimensions.ai_vegetation_unclassified_groups}=uint32"

lidar_prod/tasks/cleaning.py

@@ -6,7 +6,7 @@
 import laspy
 import pdal
 
-from lidar_prod.tasks.utils import get_pdal_reader, get_pdal_writer
+from lidar_prod.tasks.utils import get_pdal_writer, pdal_read_las_array
 
 log = logging.getLogger(__name__)
 
@@ -27,16 +27,21 @@ def __init__(self, extra_dims: Optional[Union[Iterable[str], str]]):
 
         # creating a dict where key = dimension's name and value = diemnsion's type
         #  if no "=type" in extra_dims then value = None
+        # This is for easier manipulation
         self.extra_dims_as_dict = dict()
         for extra_dim in self.extra_dims:
             if len(extra_dim.split("=")) == 2:
                 self.extra_dims_as_dict[extra_dim.split("=")[0]] = extra_dim.split("=")[1]
-            else:
+            elif extra_dim:
                 self.extra_dims_as_dict[extra_dim] = None
+            else:
+                # empty string
+                pass
 
     def get_extra_dims_as_str(self):
         """'stringify' the extra_dims list and return it, or an empty list if there is no extra dims"""
-        return_str = ",".join(self.extra_dims)
+        self.extra_dims_as_dict
+        return_str = ",".join([f"{k}={v}" for k, v in self.extra_dims_as_dict.items()])
         return return_str if return_str else []
 
     def run(self, src_las_path: str, target_las_path: str):
@@ -46,9 +51,12 @@ def run(self, src_las_path: str, target_las_path: str):
             src_las_path (str): input LAS path
             target_las_path (str): output LAS path, with specified extra dims.
         """
-        pipeline = pdal.Pipeline()
-        pipeline |= get_pdal_reader(src_las_path)
-        pipeline |= get_pdal_writer(target_las_path, extra_dims=self.get_extra_dims_as_str())
+        points = pdal_read_las_array(src_las_path)
+        # Check input dims to see what we can keep.
+        input_dims = points.dtype.fields.keys()
+        self.extra_dims_as_dict = {k: v for k, v in self.extra_dims_as_dict.items() if k in input_dims}
+
+        pipeline = pdal.Pipeline(arrays=[points]) | get_pdal_writer(target_las_path, extra_dims=self.get_extra_dims_as_str())
         os.makedirs(osp.dirname(target_las_path), exist_ok=True)
         pipeline.execute()
         log.info(f"Saved to {target_las_path}")

package_metadata.yaml

@@ -1,4 +1,4 @@
-__version__: "M11.1V1.9.1"
+__version__: "M11.1V1.9.2"
 __name__: "lidar_prod"
 __url__: "https://github.com/IGNF/lidar-prod-quality-control"
 __description__: "A 3D semantic segmentation production tool to augment rule- based Lidar classification with AI and databases."

tests/lidar_prod/tasks/test_cleaning.py

@@ -38,10 +38,11 @@ def test_cleaning_float_extra_dim():
         assert "building" not in las_dimensions
 
 
-def test_cleaning_two_float_extra_dims():
+def test_cleaning_two_float_extra_dims_and_one_fantasy_dim():
     d1 = "entropy"
     d2 = "building"
-    extra_dims = [f"{d1}=float", f"{d2}=float"]
+    d3 = "i_do_not_exist_but_no_error_incurs"
+    extra_dims = [f"{d1}=float", f"{d2}=float", f"{d3}=float"]
     cl = Cleaner(extra_dims=extra_dims)
     with tempfile.TemporaryDirectory() as td:
         clean_las_path = osp.join(td, "float_extra_dim.las")
@@ -50,10 +51,11 @@ def test_cleaning_two_float_extra_dims():
         out_a = pdal_read_las_array(clean_las_path)
         assert d1 in out_a.dtype.fields.keys()
         assert d2 in out_a.dtype.fields.keys()
+        assert d3 not in out_a.dtype.fields.keys()
 
 
 @pytest.mark.parametrize("extra_dims", ("", "entropy=float", "building=float"))
-def test_cleaning_format(extra_dims):
+def test_pdal_cleaning_format(extra_dims):
     cl = Cleaner(extra_dims=extra_dims)
     with tempfile.TemporaryDirectory() as td:
         clean_las_path = osp.join(td, "float_extra_dim.las")
@@ -69,7 +71,7 @@ def test_cleaning_format(extra_dims):
         (["entropy=float", "building=float"], "entropy=float,building=float"),
     ],
 )
-def test_cleaning_get_extra_dims_as_str(extra_dims, expected):
+def test_pdal_cleaning_get_extra_dims_as_str(extra_dims, expected):
     cleaner = Cleaner(extra_dims=extra_dims)
     assert cleaner.get_extra_dims_as_str() == expected
 
@@ -144,7 +146,7 @@ def test_cleaning_get_extra_dims_as_str(extra_dims, expected):
         ),
     ],
 )
-def test_cleaning_remove_dimensions(extra_dims, expected):
+def test_laspy_cleaning_remove_dimensions(extra_dims, expected):
     las_data = get_las_data_from_las(LAS_SUBSET_FILE_VEGETATION)
     cleaner = Cleaner(extra_dims=extra_dims)
     cleaner.remove_dimensions(las_data)
@@ -215,7 +217,7 @@ def test_cleaning_remove_dimensions(extra_dims, expected):
         ),
     ],
 )
-def test_cleaning_add_dimensions(extra_dims, expected):
+def test_laspy_cleaning_add_dimensions(extra_dims, expected):
     las_data = get_las_data_from_las(LAS_SUBSET_FILE_VEGETATION)
     cleaner = Cleaner(extra_dims=extra_dims)
     cleaner.add_dimensions(las_data)