Background check not remove (#10)

* initial switch to check only

* background class name change

* background check desc

* background check test

* rename background module

* rename background test

* background use in README

* module import background

* Increment version

* type comparison

* format changes

* flake8 fix

* sort file list

* count check

* pandas dependency

* checker removal comments

Author: tomwilsonsco

README.md

@@ -13,7 +13,7 @@ transformers (ViT) such as [Segment Anything](https://arxiv.org/abs/2304.02643).
   or standard scale before writing chips as required.
 - **Mask Segmentation**: Generate segmentation mask images from geopackage or shapefile features for supervised 
   segmentation, e.g using [U-Net](https://arxiv.org/abs/1505.04597).
-- **Remove Background Chips**: Filter out image chips containing only background. Useful for when preparing training 
+- **Check Background Chips**: Identify image chips containing only background. Useful for when preparing training 
   and testing datasets.
 
 ## Installation
@@ -86,23 +86,23 @@ image_chipper = ImageChip(
 image_chipper.chip_image()
 ```
 
-### 3. RemoveBackgroundOnly Class
-The `RemoveBackgroundOnly` class provides functionality to remove image chips that contain only background. Filtering out images only containing background helps to prepare a dataset more suitable for training models.
+### 3. CheckBackgroundOnly Class
+The `CheckBackgroundOnly` class provides functionality to list image chips that contain only background. Filtering out images only containing background helps to prepare a dataset more suitable for training models.
 
 ```python
 from rschip import RemoveBackgroundOnly
 
 # Initialize the RemoveBackgroundOnly instance
-remover = RemoveBackgroundOnly(background_val=0, non_background_min=100)
+checker = checkBackgroundOnly(background_val=0, non_background_min=1)
 
-# Remove chips with only background
-remover.remove_background_only_files(
+# Find chips with only background
+checker.check_background_chips(
     class_chips_dir="path/to/mask_directory",
     image_chips_dir="path/to/image_directory"
 )
 ```
 The default assumption is that image and mask equivalent have the same file names as shown in example 2. above. If that is
-not the case, use the `masks_prefix`, `images_prefix` arguments which are prefix strings which are removed on checking for
+not the case, use the `masks_prefix`, `images_prefix` arguments which are prefix strings which are considered on checking for
 image to mask equivalent using the bottom left (x,y) indices found in the outputs generated by `ImageChip.create_chips()`.
 
 ## License

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rschip"
-version = "0.3.3"
+version = "0.3.4"
 description = "Prepare satellite images and training data for use with deep learning models"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { text = "MIT" }
@@ -32,6 +32,7 @@ dependencies = [
     "geopandas",
     "shapely",
     "tqdm",
+    "pandas",
 ]
 requires-python = ">=3.9"
 

src/rschip/__init__.py

@@ -1,3 +1,3 @@
 from rschip.image_chip import ImageChip  # noqa: F401
 from rschip.segmentation_mask import SegmentationMask  # noqa: F401
-from rschip.remove_background_only import RemoveBackgroundOnly  # noqa: F401
+from rschip.check_background import CheckBackgroundOnly  # noqa: F401

src/rschip/check_background.py

@@ -0,0 +1,122 @@
+from pathlib import Path
+import rasterio as rio
+import numpy as np
+from typing import Optional, Union
+import pandas as pd
+
+
+class CheckBackgroundOnly:
+    """
+    Check arrays where the segmentation mask class values show background only.
+
+    This class is used to identify image chips that contain only background based on
+    their corresponding segmentation masks.
+
+    Attributes:
+        background_val (int | float): The value in the mask image array that represents the background class.
+        Defaults to 0.
+        non_background_min (int): The minimum number of non-background pixels required to consider a chip
+        as not background-only. Defaults to 1.
+    """
+
+    def __init__(
+        self, background_val: Union[int, float] = 0, non_background_min: int = 1
+    ):
+        self.background_val = background_val
+        self.non_background_min = non_background_min
+
+    @staticmethod
+    def _prefix_checker(prefix: Optional[str]) -> str:
+        return "" if prefix is None else prefix
+
+    def _find_image_eq_mask(
+        self,
+        class_chip_path: Path,
+        image_chips_dir: str,
+        masks_prefix: Optional[str],
+        images_prefix: Optional[str],
+    ) -> Path:
+        image_chips_dir = Path(image_chips_dir)
+        class_file = class_chip_path.name
+        image_file = class_file.replace(
+            self._prefix_checker(masks_prefix), self._prefix_checker(images_prefix)
+        )
+        return image_chips_dir / image_file
+
+    def check_background_only(self, class_arr: np.ndarray) -> bool:
+        """
+        Check if an image mask has fewer than the specified number of non-background pixels.
+
+        Args:
+            class_arr (numpy.ndarray): A 2D NumPy array representing the class labels for each pixel in an image mask.
+
+        Returns:
+            bool: True if the image mask has a non-background pixel count < `non_background_min`. False otherwise.
+        """
+        return np.sum(class_arr != self.background_val) < self.non_background_min
+
+    def check_background_chips(
+        self,
+        class_chips_dir: str,
+        image_chips_dir: str,
+        image_extn: str = "tif",
+        masks_prefix: Optional[str] = None,
+        images_prefix: Optional[str] = None,
+    ) -> pd.DataFrame:
+        """
+        Checks chip files to identify which ones are background only and returns a DataFrame.
+
+        This method iterates through mask chip files, checks if they are background-only,
+        and writes the results to a CSV file named 'background_only_check.csv' in the
+        `class_chips_dir`. The results are also returned as a pandas DataFrame.
+
+        The CSV file and DataFrame will contain the following columns:
+        - mask_file: The path to the mask chip file.
+        - image_file: The path to the corresponding image chip file.
+        - is_background_only: A boolean indicating if the mask is background-only.
+
+        Args:
+            class_chips_dir (str): Directory containing the chip mask image files to check.
+            image_chips_dir (str): Corresponding chip image file directory.
+            image_extn (str, optional): The extension for the image files. Defaults to "tif".
+            masks_prefix (str, optional): Prefix for mask files. Defaults to None. This prefix is removed when checking for
+            equivalent mask to image file.
+            images_prefix (str, optional): As `masks_prefix`. Prefix for image files. Defaults to None.
+
+        Returns:
+            pd.DataFrame: A DataFrame with the check results.
+
+        Raises:
+            FileNotFoundError: If no files with the specified extension are found in the input directory.
+        """
+        class_chips_dir = Path(class_chips_dir)
+        mask_files = sorted(list(class_chips_dir.glob(f"**/*.{image_extn}")))
+        if not mask_files:
+            raise FileNotFoundError(f"No {image_extn} files found in {class_chips_dir}")
+
+        print(f"Checking {len(mask_files)} files in {class_chips_dir}.")
+
+        audit_data = []
+        for f in mask_files:
+            with rio.open(f) as src:
+                img = src.read(1)
+            is_background = self.check_background_only(img)
+            image_file = self._find_image_eq_mask(
+                f, image_chips_dir, masks_prefix, images_prefix
+            )
+            audit_data.append(
+                {
+                    "mask_file": f,
+                    "image_file": image_file,
+                    "is_background_only": is_background,
+                }
+            )
+
+        df = pd.DataFrame(audit_data)
+
+        # Save to CSV
+        output_csv_path = class_chips_dir / "background_only_check.csv"
+        df.to_csv(output_csv_path, index=False)
+
+        print(f"Check results written to {output_csv_path}")
+        return df

src/rschip/remove_background_only.py

@@ -3,7 +3,8 @@
 import tempfile
 from rschip import ImageChip
 from rschip import SegmentationMask
-from rschip import RemoveBackgroundOnly
+from rschip import CheckBackgroundOnly
+import pandas as pd
 
 
 @pytest.fixture(scope="function")
@@ -37,13 +38,16 @@ def tif_files_to_list(out_dir):
     return list(Path(out_dir).glob("*.tif"))
 
 
-# repeat the same but for tif file chips not npz
-def test_tif_remove(setup_output_dir):
+def test_background_check(setup_output_dir):
     out_dir = setup_output_dir
     out_mask = out_dir / "output_mask.tif"
     out_mask_chips = out_dir / "mask_chips"
     out_img_chips = out_dir / "img_chips"
 
+    # Create directories for chips
+    out_mask_chips.mkdir()
+    out_img_chips.mkdir()
+
     mask_creator = SegmentationMask(
         "tests/data/test_img.tif", "tests/data/test_features.gpkg", out_mask
     )
@@ -62,24 +66,36 @@ def test_tif_remove(setup_output_dir):
     tif_img_files_init = len(list(out_img_chips.glob("*.tif")))
     tif_mask_files_init = len(list(out_mask_chips.glob("*.tif")))
 
-    remover = RemoveBackgroundOnly(background_val=0, non_background_min=100)
+    checker = CheckBackgroundOnly(background_val=0, non_background_min=100)
 
-    remover.remove_background_only_files(out_mask_chips, out_img_chips)
+    df = checker.check_background_chips(str(out_mask_chips), str(out_img_chips))
 
     tif_img_files_final = len(list(out_img_chips.glob("*.tif")))
     tif_mask_files_final = len(list(out_mask_chips.glob("*.tif")))
 
-    assert tif_img_files_final < tif_img_files_init, "No img files were removed"
+    # Check that no files were removed
+    assert tif_img_files_final == tif_img_files_init, "Image files were removed"
+    assert tif_mask_files_final == tif_mask_files_init, "Mask files were removed"
 
-    assert tif_mask_files_final < tif_mask_files_init, "No chip files were removed"
+    # Check that CSV was created
+    csv_path = out_mask_chips / "background_only_check.csv"
+    assert csv_path.exists(), "Audit CSV file was not created."
 
-    assert (
-        tif_mask_files_final == tif_img_files_final
-    ), "Remaining chips and image file number differs"
+    # Check DataFrame and CSV content
+    assert len(df) == tif_mask_files_init
+    assert "is_background_only" in df.columns
+    assert df[
+        "is_background_only"
+    ].any(), "Expected some background-only chips to be True."
+    assert not df[
+        "is_background_only"
+    ].all(), "Expected some non-background chips to be False."
 
+    csv_df = pd.read_csv(csv_path)
+    assert len(csv_df) == len(df)
 
-# specifying a large and small background threshold and compare
-def test_non_background_min(setup_output_dir):
+
+def test_non_background_min_check(setup_output_dir):
     out_dir = setup_output_dir
     out_mask = out_dir / "output_mask.tif"
     out_mask_chips = out_dir / "mask_chips"
@@ -99,22 +115,16 @@ def test_non_background_min(setup_output_dir):
         input_image_path=out_mask,
         output_name="test_img",
     )
-    remover = RemoveBackgroundOnly(background_val=0, non_background_min=1)
-    remover.remove_background_only_files(out_mask_chips, out_img_chips)
-
-    tif_img_files_final1 = len(list(out_img_chips.glob("*.tif")))
-    tif_mask_files_final1 = len(list(out_mask_chips.glob("*.tif")))
+    # Audit with a low threshold
+    checker1 = CheckBackgroundOnly(background_val=0, non_background_min=1)
+    df1 = checker1.check_background_chips(str(out_mask_chips), str(out_img_chips))
+    background_only_count1 = df1["is_background_only"].sum()
 
-    remover = RemoveBackgroundOnly(background_val=0, non_background_min=10000)
-    remover.remove_background_only_files(out_mask_chips, out_img_chips)
-
-    tif_img_files_final2 = len(list(out_img_chips.glob("*.tif")))
-    tif_mask_files_final2 = len(list(out_mask_chips.glob("*.tif")))
-
-    assert (
-        tif_img_files_final2 < tif_img_files_final1
-    ), "Non background threshold on images failed"
+    # Audit with a high threshold
+    checker2 = CheckBackgroundOnly(background_val=0, non_background_min=10000)
+    df2 = checker2.check_background_chips(str(out_mask_chips), str(out_img_chips))
+    background_only_count2 = df2["is_background_only"].sum()
 
     assert (
-        tif_mask_files_final2 < tif_mask_files_final1
-    ), "Non background threshold on masks failed"
+        background_only_count2 >= background_only_count1
+    ), "Higher non_background_min threshold should result in more background-only chips."