Remove npz output (#9)

* remove doc npz references

* remove npz references in ImageChip

* remove npz methods from RemoveBackgroundOnly

* test references to npz removed

* Increment patch version

* formatting fix

* imports and formatting

Author: tomwilsonsco

README.md

@@ -40,7 +40,6 @@ image_chipper = ImageChip(
     output_path="path/to/output_directory_image",
     pixel_dimensions=128,
     offset=64,
-    output_format="tif",
 )
 
 # set a min max normaliser 
@@ -50,9 +49,8 @@ image_chipper.set_normaliser(min_val=500, max_val=3000)
 # Generate chips
 image_chipper.chip_image()
 ```
-With the `output_format` parameter set to `"tif"`, each resulting tile is named using a suffix that represents the bottom left `(x, y)`
-pixel coordinate position. If output_format is set to `"npz"`, the resulting .npz zip file contains a dictionary of arrays, 
-where the keys are the same as these tile names. By default, the prefix of each tile name is taken from the input image file name 
+Each resulting tile is named using a suffix that represents the bottom left `(x, y)`
+pixel coordinate position. By default, the prefix of each tile name is taken from the input image file name 
 (`input_image_path`), unless you specify `output_name`.
 
 Using the parameter `use_multiprocessing=True` (default) makes chipping process faster by using multiple cores. 
@@ -84,13 +82,12 @@ image_chipper = ImageChip(
     output_name="large_image",
     pixel_dimensions=128,
     offset=64,
-    output_format="tif",
 )
 image_chipper.chip_image()
 ```
 
 ### 3. RemoveBackgroundOnly Class
-The `RemoveBackgroundOnly` class provides functionality to remove image chips (either could be tifs or numpy arrays inside npz file) that contain only background. Filtering out images only containing background helps to prepare a dataset more suitable for training models.
+The `RemoveBackgroundOnly` class provides functionality to remove image chips that contain only background. Filtering out images only containing background helps to prepare a dataset more suitable for training models.
 
 ```python
 from rschip import RemoveBackgroundOnly

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rschip"
-version = "0.3.2"
+version = "0.3.3"
 description = "Prepare satellite images and training data for use with deep learning models"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { text = "MIT" }

src/rschip/image_chip.py

@@ -20,9 +20,6 @@ class ImageChip:
         pixel_dimensions (int): The height and width of each tile in pixels. Defaults to 128.
         offset (int): The offset used when creating tiles, i.e. the step size. Defaults to 64.
         use_multiprocessing (bool): Whether to use multiprocessing for chipping. Defaults to True.
-        output_format (str): The format of the output files, either 'tif' or 'npz'.
-                             If tif then tif file written per tile window. If npz then `batch_size` batches
-                              of array tiles are written into one npz file. Defaults to tif.
         max_batch_size (int): The maximum number of tiles to process in a batch.
                              If multiprocessing is enabled, the actual batch size may be less. Defaults to 1000.
     """
@@ -35,7 +32,6 @@ def __init__(
         pixel_dimensions=128,
         offset=64,
         use_multiprocessing=True,
-        output_format="tif",
         max_batch_size=1000,
     ):
 
@@ -47,7 +43,6 @@ def __init__(
         self.standard_scaler = None
         self.normaliser = None
         self.use_multiprocessing = use_multiprocessing
-        self.output_format = output_format
         self.max_batch_size = max_batch_size
         if not self.input_image_path.exists():
             raise FileNotFoundError(f"Input image not found: {self.input_image_path}")
@@ -56,10 +51,6 @@ def __init__(
             raise ValueError("pixel_dimensions must be a positive integer")
         if self.offset <= 0:
             raise ValueError("offset must be a positive integer")
-        if self.output_format not in ("tif", "npz"):
-            raise ValueError(
-                f"output_format must be 'tif' or 'npz', got '{self.output_format}'"
-            )
 
     def _read_image_metadata(self) -> None:
         """
@@ -133,22 +124,6 @@ def _save_chip(self, chip, transform, output_file_path, d_type, src) -> None:
         ) as dst:
             dst.write(chip)
 
-    def _save_batch_as_npz(self, batch, batch_index) -> None:
-        """
-        Save a batch of chips as an NPZ file.
-
-        Args:
-            batch (dict): Dictionary containing image chips.
-            batch_index (int): Index of the batch.
-
-        Returns:
-            None: Writes the batch to an NPZ file at the specified path.
-        """
-        output_file_path = self.output_path / f"batch_{batch_index}.npz"
-        if output_file_path.exists():
-            output_file_path.unlink()
-        np.savez_compressed(output_file_path, **batch)
-
     def _output_file(self, x: int, y: int) -> Path:
         """
         Generate the output file path for a chip based on its x and y coordinates.
@@ -428,8 +403,7 @@ def _process_batch(self, batch_vals):
         Returns:
             None
         """
-        batch_id, batch = batch_vals
-        out = {}
+        _, batch = batch_vals
         with rio.open(self.input_image_path) as src:
             for x, y, window in batch:
                 chip = src.read(
@@ -440,16 +414,9 @@ def _process_batch(self, batch_vals):
                 if self.normaliser:
                     chip = self.apply_normaliser(chip, self.normaliser)
 
-                if self.output_format == "tif":
-                    output_file_path = self._output_file(x, y)
-                    transform = src.window_transform(window)
-                    self._save_chip(chip, transform, output_file_path, chip.dtype, src)
-                elif self.output_format == "npz":
-                    arr_name = f"{self.output_name}_{x}_{y}"
-                    out[arr_name] = chip
-
-        if self.output_format == "npz" and out:
-            self._save_batch_as_npz(out, batch_id)
+                output_file_path = self._output_file(x, y)
+                transform = src.window_transform(window)
+                self._save_chip(chip, transform, output_file_path, chip.dtype, src)
 
     def _calculate_batches(self, windows):
         """
@@ -486,7 +453,7 @@ def chip_image(self) -> None:
         Method uses rasterio to read a satellite image, then splits the image into
         smaller square tiles of specified dimensions and saves them to the output path.
 
-        The output tile file names are suffixed with x and y offsets and saved as TIFF files or NPZ files.
+        The output tile file names are suffixed with x and y offsets and saved as TIFF files.
         Optionally the chip pixel values can be standard scaled before saving, using a sample of the full image pixels.
 
         Returns:

src/rschip/remove_background_only.py

@@ -35,19 +35,6 @@ def _find_image_eq_mask(
         )
         return image_chips_dir / image_file
 
-    def _find_img_npz_eq_mask(self, class_npz_file: Path, image_npz_dir: str) -> Path:
-        image_npz_dir = Path(image_npz_dir)
-        class_file = class_npz_file.name
-        return image_npz_dir / class_file
-
-    def _find_img_key_from_mask_key(
-        self, class_key: str, masks_prefix: Optional[str], images_prefix: Optional[str]
-    ) -> str:
-        img_key = class_key.replace(
-            self._prefix_checker(masks_prefix), self._prefix_checker(images_prefix)
-        )
-        return img_key
-
     def check_background_only(self, class_arr: np.ndarray) -> bool:
         """
         Check if an image mask has more than the specified number of non-background pixels.
@@ -106,57 +93,3 @@ def remove_background_only_files(
 
         image_files = list(class_chips_dir.glob(f"**/*.{image_extn}"))
         print(f"{len(image_files)} in {class_chips_dir} after.")
-
-    def remove_background_only_npz(
-        self,
-        class_npz_dir: str,
-        image_npz_dir: str,
-        masks_prefix: Optional[str] = None,
-        images_prefix: Optional[str] = None,
-    ) -> None:
-        """
-        Remove arrays from NPZ files where the mask contains background only.
-
-        Args:
-            class_npz_dir (str): Directory containing the chip mask NPZ files to check.
-            image_npz_dir (str): Corresponding chip image NPZ file directory - if mask is all background, image is removed too.
-            masks_prefix (str, optional): Prefix for mask files. Defaults to None. This prefix is removed when checking for
-            equivalent mask to image file.
-            images_prefix (str, optional): As `masks_prefix`. Prefix for image files. Defaults to None.
-
-        Raises:
-            FileNotFoundError: If no NPZ files are found in the input directory.
-        """
-        class_npz_dir = Path(class_npz_dir)
-        npz_files = list(class_npz_dir.glob("**/*.npz"))
-        if not npz_files:
-            raise FileNotFoundError(f"No npz files found in {class_npz_dir}")
-
-        for f in npz_files:
-            npz_class_dict = np.load(f)
-            img_npz_file = self._find_img_npz_eq_mask(f, image_npz_dir)
-            npz_image_dict = np.load(img_npz_file)
-
-            out_class_dict = {}
-            out_img_dict = {}
-
-            print(f"{f.name} initially {len(npz_class_dict.keys())}...")
-            for key in npz_class_dict.files:
-                class_arr = npz_class_dict[key]
-                if not self.check_background_only(class_arr):
-                    out_class_dict[key] = class_arr
-                    img_key = self._find_img_key_from_mask_key(
-                        key, masks_prefix, images_prefix
-                    )
-                    out_img_dict[img_key] = npz_image_dict[img_key]
-            print(f"{f.name} finally {len(out_class_dict.keys())}...")
-            npz_class_dict.close()
-            npz_image_dict.close()
-            f.unlink()
-            img_npz_file.unlink()
-
-            if out_class_dict:
-                np.savez_compressed(f, **out_class_dict)
-                np.savez_compressed(img_npz_file, **out_img_dict)
-            else:
-                print(f"No valid entries left in {f.name}, deleting the NPZ file.")

tests/test_image_chip.py

@@ -21,7 +21,6 @@ def chip_image_run(
     pixel_dimensions=128,
     offset=64,
     use_multiprocessing=True,
-    output_format="tif",
     max_batch_size=10,
     scaler=False,
     normaliser=False,
@@ -32,7 +31,6 @@ def chip_image_run(
         pixel_dimensions=pixel_dimensions,
         offset=offset,
         use_multiprocessing=use_multiprocessing,
-        output_format=output_format,
         max_batch_size=max_batch_size,
     )
     if scaler:
@@ -43,106 +41,17 @@ def chip_image_run(
     image_chip.chip_image()
 
 
-def load_npz(npz_file_path):
-    with np.load(npz_file_path) as data:
-        test_key = data.files[0]
-        arr = data[test_key]
-    return test_key, arr
-
-
 def load_tif(tif_file_path):
     with rio.open(tif_file_path) as f:
         arr = f.read()
         prof = f.profile
     return arr, prof
 
 
-def npz_files_to_list(out_dir):
-    array_list = []
-    for file_path in Path(out_dir).glob("*.npz"):
-        with np.load(file_path) as data:
-            for array_name in data.files:
-                array_list.append(data[array_name])
-    return array_list
-
-
 def tif_files_to_list(out_dir):
     return list(Path(out_dir).glob("*.tif"))
 
 
-def test_image_chip(setup_output_dir):
-    out_dir = setup_output_dir
-    # Test chipping with TIFF output
-    chip_image_run(output_path=out_dir)
-
-    # Verify that TIFF files were created
-    tif_files = list(out_dir.glob("*.tif"))
-    assert len(tif_files) > 0, "No TIFF files were created."
-
-    # Test chipping with NPZ output
-    chip_image_run(output_path=out_dir, output_format="npz")
-
-    # Verify that NPZ files were created
-    npz_files = list(out_dir.glob("*.npz"))
-    assert len(npz_files) > 0, "No NPZ files were created."
-
-
-def test_array_equality(setup_output_dir):
-    out_dir = setup_output_dir
-    chip_image_run(output_path=out_dir, output_format="tif")
-    chip_image_run(output_path=out_dir, output_format="npz")
-
-    # Load one NPZ and corresponding TIFF file to compare arrays
-    test_key, npz_arr = load_npz(out_dir / "batch_0.npz")
-    tif_arr, _ = load_tif(f"{out_dir}/{test_key}.tif")
-
-    # Compare arrays
-    assert np.array_equal(tif_arr, npz_arr), "Arrays are not equal."
-
-
-def test_image_and_array_count(setup_output_dir):
-    out_dir = setup_output_dir
-
-    chip_image_run(output_path=out_dir, output_format="tif")
-    chip_image_run(output_path=out_dir, output_format="npz")
-
-    # Compare the number of TIFF images and NPZ arrays
-    npz_array_count = len(npz_files_to_list(out_dir))
-    tif_file_count = len(tif_files_to_list(out_dir))
-    assert npz_array_count == tif_file_count, "Mismatch in number of arrays and images."
-
-
-def test_scaler_functionality():
-    image_chip = ImageChip(
-        input_image_path="tests/data/test_img.tif", output_path="tmp"
-    )
-    sample_array = np.random.rand(3, 128, 128) * 100
-    scaler_dict = {
-        0: {"mean": 50, "std": 10},
-        1: {"mean": 60, "std": 15},
-        2: {"mean": 40, "std": 20},
-    }
-    scaled_array = image_chip.apply_scaler(sample_array, scaler_dict)
-    unscaled_array = ImageChip.unapply_scaler(scaled_array, scaler_dict)
-    assert np.allclose(
-        sample_array, unscaled_array, atol=1e-1
-    ), "Scaling/unscaling mismatch"
-
-
-def test_large_window_image(setup_output_dir):
-    out_dir = setup_output_dir
-    chip_image_run(
-        output_path=out_dir, output_format="tif", pixel_dimensions=512, offset=256
-    )
-    chip_image_run(
-        output_path=out_dir, output_format="npz", pixel_dimensions=512, offset=256
-    )
-
-    # Verify that at least one file was created
-    tif_files = tif_files_to_list(out_dir)
-    assert len(tif_files) > 0, "No TIFF files were created for large window."
-
-
 def test_multiprocessor_not(setup_output_dir):
     out_dir = setup_output_dir
     chip_image_run(output_path=out_dir, use_multiprocessing=True)
@@ -180,7 +89,6 @@ def test_tile_count(setup_output_dir):
         input_image_path=input_image_path,
         pixel_dimensions=pixel_dimensions,
         offset=offset,
-        output_format="tif",
     )
 
     # Verify that the expected number of tiles were created
@@ -193,7 +101,7 @@ def test_tile_count(setup_output_dir):
 def test_normalising(setup_output_dir):
     out_dir = setup_output_dir
 
-    chip_image_run(output_path=out_dir, output_format="tif", normaliser=True)
+    chip_image_run(output_path=out_dir, normaliser=True)
 
     with rio.open(tif_files_to_list(out_dir)[0]) as f:
         test_array = f.read()
@@ -204,7 +112,7 @@ def test_normalising(setup_output_dir):
 def test_standard_scaling(setup_output_dir):
     out_dir = setup_output_dir
 
-    chip_image_run(output_path=out_dir, output_format="tif", scaler=True)
+    chip_image_run(output_path=out_dir, scaler=True)
 
     with rio.open(tif_files_to_list(out_dir)[0]) as f:
         test_array = f.read()
@@ -371,13 +279,14 @@ def test_init_invalid_offset():
         )
 
 
-def test_init_invalid_output_format():
-    with pytest.raises(ValueError, match="output_format must be 'tif' or 'npz'"):
-        ImageChip(
-            input_image_path="tests/data/test_img.tif",
-            output_path="tmp",
-            output_format="jpg",
-        )
+def test_image_chip(setup_output_dir):
+    out_dir = setup_output_dir
+    # Test chipping with TIFF output
+    chip_image_run(output_path=out_dir)
+
+    # Verify that TIFF files were created
+    tif_files = list(out_dir.glob("*.tif"))
+    assert len(tif_files) > 0, "No TIFF files were created."
 
 
 def test_nodata_val_set_from_image():