Add CLI script for data conversion (#13)

* Add CLI script for data conversion

* Add pyproject.toml and filter max-projection tifs from glob

* Fix tests

Author: constantinpape

.github/workflows/test.yml

@@ -23,9 +23,9 @@ jobs:
         with:
           environment-file: environment.yaml
 
-      # - name: Install package
-      #   shell: bash -l {0}
-      #   run: pip install -e .
+      - name: Install package
+        shell: bash -l {0}
+        run: pip install -e .
 
       - name: Run tests
         shell: bash -l {0}

README.md

@@ -4,6 +4,7 @@ Data processing for light-sheet microscopy, specifically for data from [Flamingo
 
 The `flamingo_tools` library implements functionality for:
 - converting the lightsheet data into a format compatible with [BigDataViewer](https://imagej.net/plugins/bdv/) and [BigStitcher](https://imagej.net/plugins/bigstitcher/).
+- Cell / nucleus segmentation via a 3D U-net.
 - ... and more functionality is planned!
 
 This is work in progress!
@@ -12,25 +13,38 @@ This is work in progress!
 ## Requirements & Installation
 
 You need a python environment with the following dependencies: [pybdv](https://github.com/constantinpape/pybdv) and [z5py](https://github.com/constantinpape/z5).
-You can for example install these dependencies with [mamba](https://github.com/mamba-org/mamba) (a faster implementation of [conda](https://docs.conda.io/en/latest/)) via: 
+You install these dependencies with [mamba](https://github.com/mamba-org/mamba) or [conda](https://docs.conda.io/en/latest/) via: 
 ```bash
-$ mamba install -c conda-forge z5py pybdv
+conda install -c conda-forge z5py pybdv
 ```
-You can also set up a new environment with these dependencies using the file `environment.yaml`:
+(for an existing conda environment). You can also set up a new environment with all required dependencies using the file `environment.yaml`:
 ```bash
-$ mamba env create -f environment.yaml
+conda env create -f environment.yaml
+```
+This will create the environment `flamingo`, which you can then activate via `conda activate flamingo`.
+Finally, to install `flamingo_tools` into the environment run
+```bash
+pip install -e .
 ```
 
 ## Usage
 
-We provide the follwoing scripts:
+We provide a command line tool, `convert_flamingo`, for converting data from the flamingo microscope to a data format compatible with BigDataViewer / BigStitcher:
+```bash
+convert_flamingo -i /path/to/data -o /path/to/output.n5 --file_ext .tif
+```
+Here, `/path/to/data` is the filepath to the folder with the flamingo data to be converted, `/path/to/output.n5` is the filepath where the converted data will be stored, and `--file_ext .tif` declares that the files are stored as tif stacks.
+Use `--file_ext .raw` isntead if the data is stored in raw files.
+
+The data will be converted to the [bdv.n5 format](https://github.com/bigdataviewer/bigdataviewer-core/blob/master/BDV%20N5%20format.md).
+It can be opened with BigDataViewer via `Plugins->BigDataViewer->Open XML/HDF5`.
+Or with BigStitcher as described [here](https://imagej.net/plugins/bigstitcher/open-existing).
+
+You can also check out the following example scripts:
 - `create_synthetic_data.py`: create small synthetic test data to check that the scripts work. 
-- `convert_flamingo_data.py`: convert flamingo data to a file format comatible with BigDataViewer / BigStitcher via command line interface. Run `python convert_flamingo_data.py -h` for details. 
 - `convert_flamingo_data_examples.py`: convert flamingo data to a file format comatible with BigDataViewer / BigStitcher with parameters defined in the python script. Contains two example functions:
     - `convert_synthetic_data` to convert the synthetic data created via `create_synthetic_data.py`.
     - `convert_flamingo_data_moser` to convert sampled flamingo data from the Moser group.
 - `load_data.py`: Example script for how to load sub-regions from the converted data into python.
 
-The data will be converted to the [bdv.n5 format](https://github.com/bigdataviewer/bigdataviewer-core/blob/master/BDV%20N5%20format.md).
-It can be opened with BigDataViewer via `Plugins->BigDataViewer->Open XML/HDF5`.
-Or with BigStitcher as described [here](https://imagej.net/plugins/bigstitcher/open-existing).
+For advanced examples to segment data with a U-Net, check out the `scripts` folder.

convert_flamingo_data.py

@@ -1,4 +1,4 @@
-name: lightsheet
+name: flamingo
 
 channels:
   - pytorch

environment.yaml

@@ -300,6 +300,9 @@ def convert_lightsheet_to_bdv(
         convert_to_ome_zarr = True
 
     files = sorted(glob(os.path.join(root, f"**/*{file_ext}"), recursive=True))
+    if file_ext == ".tif":
+        # We need to exlcude the max-projetion tifs that are saved alongside the tifs.
+        files = [ff for ff in files if "_MP.tif" not in ff]
     # Raise an error if we could not find any files.
     if len(files) == 0:
         raise ValueError(f"Could not find any files in {root} with extension {file_ext}.")
@@ -409,9 +412,14 @@ def convert_lightsheet_to_bdv_cli():
     )
 
     args = parser.parse_args()
+    if args.metadata_pattern == "":
+        metadata_pattern = None
+    else:
+        metadata_pattern = args.metadata_pattern
+
     convert_lightsheet_to_bdv(
         root=args.input_root,
         out_path=args.out_path,
         file_ext=args.file_ext,
-        metadata_file_name_pattern=args.metadata_pattern
+        metadata_file_name_pattern=metadata_pattern
     )

flamingo_tools/data_conversion.py

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=64.0", "wheel"]
+build-backend = "setuptools.build_meta"

pyproject.toml

@@ -1,9 +1,16 @@
 import runpy
 from setuptools import setup, find_packages
 
-version = runpy.run_path('flamingo_tools/version.py')['__version__']
-setup(name='flamingo_tools',
-      packages=find_packages(exclude=['test']),
-      version=version,
-      author='Constantin Pape',
-      license='MIT')
+version = runpy.run_path("flamingo_tools/version.py")["__version__"]
+setup(
+    name="flamingo_tools",
+    packages=find_packages(exclude=["test"]),
+    version=version,
+    author="Constantin Pape",
+    license="MIT",
+    entry_points={
+        "console_scripts": [
+            "convert_flamingo = flamingo_tools.data_conversion:convert_lightsheet_to_bdv_cli"
+        ]
+    }
+)

setup.py

@@ -0,0 +1,35 @@
+import os
+import unittest
+from shutil import rmtree
+
+import z5py
+
+from subprocess import run
+
+
+class TestCLI(unittest.TestCase):
+    folder = "./tmp"
+
+    def setUp(self):
+        from flamingo_tools import create_test_data
+
+        # TODO Create flamingo metadata.
+        create_test_data(self.folder)
+
+    def tearDown(self):
+        rmtree(self.folder)
+
+    def test_convert_flamingo(self):
+        out_path = os.path.join(self.folder, "converted_data.n5")
+        cmd = ["convert_flamingo", "-i", self.folder, "-o", out_path, "--metadata_pattern", ""]
+        run(cmd)
+
+        self.assertTrue(os.path.exists(out_path))
+        xml_path = out_path.replace(".n5", ".xml")
+        self.assertTrue(os.path.exists(xml_path))
+        with z5py.File(out_path, "r") as f:
+            self.assertTrue("setup0" in f)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_cli.py

@@ -1,18 +1,15 @@
 import os
-import sys
 import unittest
 from shutil import rmtree
 
-sys.path.append("..")
-
 
 class TestDataConversion(unittest.TestCase):
     folder = "./tmp"
 
     def setUp(self):
         from flamingo_tools import create_test_data
 
-        # TODO metadata
+        # TODO Create flamingo metadata.
         create_test_data(self.folder)
 
     def tearDown(self):
@@ -22,7 +19,7 @@ def test_convert_lightsheet_to_bdv(self):
         from flamingo_tools import convert_lightsheet_to_bdv
 
         out_path = os.path.join(self.folder, "converted_data.n5")
-        convert_lightsheet_to_bdv(self.folder, out_path=out_path)
+        convert_lightsheet_to_bdv(self.folder, out_path=out_path, metadata_file_name_pattern=None)
 
         self.assertTrue(os.path.exists(out_path))
         xml_path = out_path.replace(".n5", ".xml")