diff --git a/.gitignore b/.gitignore index d25c890..7124fe7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ synthetic_data/ __pycache__/ converted/ +*.egg-info/ diff --git a/flamingo_tools/data_conversion.py b/flamingo_tools/data_conversion.py index ac7ef89..ca4d421 100644 --- a/flamingo_tools/data_conversion.py +++ b/flamingo_tools/data_conversion.py @@ -1,5 +1,6 @@ import multiprocessing as mp import os +import re from glob import glob from pathlib import Path @@ -54,19 +55,15 @@ def _read_start_position_flamingo(path): return start_position -def read_metadata_flamingo(metadata_paths, center_tiles): - start_positions = [] +def read_metadata_flamingo(metadata_path, offset=None): resolution, unit = None, None - for path in metadata_paths: - resolution, unit = _read_resolution_and_unit_flamingo(path) - start_position = _read_start_position_flamingo(path) - start_positions.append(start_position) - start_positions = np.array(start_positions) - offset = np.min(start_positions, axis=0) if center_tiles else np.array([0.0, 0.0, 0.0]) + resolution, unit = _read_resolution_and_unit_flamingo(metadata_path) + start_position = _read_start_position_flamingo(metadata_path) def _pos_to_trafo(pos): - pos -= offset + if offset is not None: + pos -= offset # FIXME: dirty hack # scale = 4 @@ -93,11 +90,9 @@ def _pos_to_trafo(pos): } return trafo - transformations = [ - _pos_to_trafo(pos) for pos in start_positions - ] + transformation = _pos_to_trafo(start_position) # We have to reverse the resolution because pybdv expects ZYX. - return resolution[::-1], unit, transformations + return resolution[::-1], unit, transformation # TODO derive the scale factors from the shape rather than hard-coding it to 5 levels @@ -106,15 +101,61 @@ def derive_scale_factors(shape): return scale_factors +def flamingo_filename_parser(file_path, name_mapping): + filename = os.path.basename(file_path) + + # Extract the timepoint. + match = re.search(r'_t(\d+)_', filename) + if match: + timepoint = int(match.group(1)) + else: + timepoint = 0 + + # Extract the additional attributes. + attributes = {} + if name_mapping is None: + name_mapping = {} + + # Extract the channel. + match = re.search(r'_C(\d+)_', filename) + channel = int(match.group(1)) if match else 0 + channel_mapping = name_mapping.get("channel", {}) + attributes["channel"] = {"id": channel, "name": channel_mapping.get(channel, str(channel))} + + # Extract the tile. + match = re.search(r'_R(\d+)_', filename) + tile = int(match.group(1)) if match else 0 + tile_mapping = name_mapping.get("tile", {}) + attributes["tile"] = {"id": tile, "name": tile_mapping.get(tile, str(tile))} + + # Extract the illumination. + match = re.search(r'_I(\d+)_', filename) + illumination = int(match.group(1)) if match else 0 + illumination_mapping = name_mapping.get("illumination", {}) + attributes["illumination"] = {"id": illumination, "name": illumination_mapping.get(illumination, str(illumination))} + + # Extract D. TODO what is this? + match = re.search(r'_D(\d+)_', filename) + D = int(match.group(1)) if match else 0 + D_mapping = name_mapping.get("D", {}) + attributes["D"] = {"id": D, "name": D_mapping.get(D, str(D))} + + # BDV also supports an angle attribute, but it does not seem to be stored in the filename + # "angle": {"id": 0, "name": "0"} + + attribute_id = f"c{channel}-t{tile}-i{illumination}-d{D}" + return timepoint, attributes, attribute_id + + def convert_lightsheet_to_bdv( root: str, - channel_folders: Dict[str, str], - image_file_name_pattern: str, out_path: str, + attribute_parser: callable = flamingo_filename_parser, + attribute_names: Optional[Dict[str, Dict[int, str]]] = None, metadata_file_name_pattern: Optional[str] = None, metadata_root: Optional[str] = None, metadata_type: str = "flamingo", - center_tiles: bool = True, + center_tiles: bool = False, resolution: Optional[List[float]] = None, unit: Optional[str] = None, scale_factors: Optional[List[List[int]]] = None, @@ -125,24 +166,14 @@ def convert_lightsheet_to_bdv( The data is converted to the bdv-n5 file format and can be opened with BigDataViewer or BigStitcher. This function is written with data layout and metadata of flamingo microscopes in mind, but could potentially be adapted to other data formats. - We currently don't support multiple timepoints, but support can be added if needed. - This function assumes the following input data format: - //.tif - /.tif - /... - //.tif - /.tif - /... + TODO explain the attribute parsing. Args: - root: Folder that contains the folders with tifs for each channel. - channel_folders: Dictionary that maps the name of each channel to the corresponding folder name - underneath the root folder. - image_file_name_pattern: The pattern for the names of the tifs that contain the data. - This expects a glob pattern (name with '*') to select the corresponding tif files . - The simplest pattern that should work in most cases is '*.tif'. + root: Folder that contains the image data stored as tifs. + This function will take into account all tif files in folders beneath this root directory. out_path: Output path where the converted data is saved. + attribute_parser: TODO metadata_file_name_pattern: The pattern for the names of files that contain the metadata. For flamingo metadata the following pattern should work: '*_Settings.txt'. metadata_root: Different root folder for the metadata. By default 'root' is used here as well. @@ -170,18 +201,44 @@ def convert_lightsheet_to_bdv( if ext == "": out_path = str(Path(out_path).with_suffix(".n5")) - # Iterate over the channels - for channel_id, (channel_name, channel_folder) in enumerate(channel_folders.items()): - - # Get all the image file paths for this channel. - tile_pattern = os.path.join(root, channel_folder, image_file_name_pattern) - file_paths = sorted(glob(tile_pattern)) - assert len(file_paths) > 0, tile_pattern + files = sorted(glob(os.path.join(root, "**/*.tif"), recursive=True)) + if metadata_file_name_pattern is None: + metadata_files = [None] * len(files) + offset = None + else: + metadata_files = sorted( + glob( + os.path.join(root if metadata_root is None else metadata_root, f"**/{metadata_file_name_pattern}"), + recursive=True + ) + ) + assert len(metadata_files) == len(files) + + if center_tiles: + start_positions = [] + for mpath in metadata_files: + start_positions.append(_read_start_position_flamingo(mpath)) + offset = np.min(start_positions, axis=0) + else: + offset = None + + next_setup_id = 0 + attrs_to_setups = {} + + for file_path, metadata_file in zip(files, metadata_files): + timepoint, attributes, aid = attribute_parser(file_path, attribute_names) + + if aid in attrs_to_setups: + setup_id = attrs_to_setups[aid] + else: + attrs_to_setups[aid] = next_setup_id + setup_id = next_setup_id + next_setup_id += 1 # Read the metadata if it was given. - if metadata_file_name_pattern is None: # No metadata given. + if metadata_file is None: # No metadata given. # We don't use any tile transformation. - tile_transformations = [None] * len(file_paths) + tile_transformation = None # Set resolution and unit to their default values if they were not passed. if resolution is None: resolution = [1.0, 1.0, 1.0] @@ -189,41 +246,28 @@ def convert_lightsheet_to_bdv( unit = "pixel" else: # We have metadata and read it. - metadata_pattern = os.path.join( - root if metadata_root is None else metadata_root, - channel_folder, metadata_file_name_pattern - ) - metadata_paths = sorted(glob(metadata_pattern)) - assert len(metadata_paths) == len(file_paths) - resolution, unit, tile_transformations = read_metadata_flamingo(metadata_paths, center_tiles) - - if channel_name is None or channel_name.strip() == "": #channel name is empty, assign channel id as name - channel_name = str(channel_id) - - for tile_id, (file_path, tile_transformation) in enumerate(zip(file_paths, tile_transformations)): - - # Try to memmap the data. If that doesn't work fall back to loading it into memory. - try: - data = tifffile.memmap(file_path, mode="r") - except ValueError: - print(f"Could not memmap the data from {file_path}. Fall back to load it into memory.") - data = tifffile.imread(file_path) - - print("Converting channel", channel_id, "tile", tile_id, "from", file_path, "with shape", data.shape) - if scale_factors is None: - scale_factors = derive_scale_factors(data.shape) - - pybdv.make_bdv( - data, out_path, - downscale_factors=scale_factors, downscale_mode="mean", - n_threads=n_threads, - resolution=resolution, unit=unit, - attributes={ - "channel": {"id": channel_id, "name": channel_name}, "tile": {"id": tile_id, "name": str(tile_id)}, - "angle": {"id": 0, "name": "0"}, "illumination": {"id": 0, "name": "0"} - }, - affine=tile_transformation, - ) + resolution, unit, tile_transformation = read_metadata_flamingo(metadata_file, offset) + + print(f"Converting tp={timepoint}, channel={attributes['channel']}, tile={attributes['tile']}") + try: + data = tifffile.memmap(file_path, mode="r") + except ValueError: + print(f"Could not memmap the data from {file_path}. Fall back to load it into memory.") + data = tifffile.imread(file_path) + + if scale_factors is None: + scale_factors = derive_scale_factors(data.shape) + + pybdv.make_bdv( + data, out_path, + downscale_factors=scale_factors, downscale_mode="mean", + n_threads=n_threads, + resolution=resolution, unit=unit, + attributes=attributes, + affine=tile_transformation, + timepoint=timepoint, + setup_id=setup_id, + ) # TODO expose more arguments via CLI. diff --git a/flamingo_tools/mobie.py b/flamingo_tools/mobie.py new file mode 100644 index 0000000..fc5091d --- /dev/null +++ b/flamingo_tools/mobie.py @@ -0,0 +1,77 @@ +import os +import tempfile +from typing import Tuple + +from mobie import add_bdv_image, add_segmentation +from mobie.metadata.dataset_metadata import read_dataset_metadata + + +# TODO refactor to mobie utils +def _source_exists(mobie_project, mobie_dataset, source_name): + dataset_folder = os.path.join(mobie_project, mobie_dataset) + metadata = read_dataset_metadata(dataset_folder) + sources = metadata.get("sources", {}) + return source_name in sources + + +def add_raw_to_mobie( + mobie_project: str, + mobie_dataset: str, + source_name: str, + xml_path: str, + skip_existing: bool = True, + setup_id: int = 0, +): + """ + """ + # Check if we have converted this data already. + have_source = _source_exists(mobie_project, mobie_dataset, source_name) + if have_source and skip_existing: + print(f"Source {source_name} already exists in {mobie_project}:{mobie_dataset}.") + print("Conversion to mobie will be skipped.") + return + elif have_source: + raise NotImplementedError + + with tempfile.TemporaryDirectory() as tmpdir: + add_bdv_image( + xml_path=xml_path, + root=mobie_project, + dataset_name=mobie_dataset, + image_name=source_name, + tmp_folder=tmpdir, + file_format="bdv.n5", + setup_ids=[setup_id], + ) + + +def add_segmentation_to_mobie( + mobie_project: str, + mobie_dataset: str, + source_name: str, + segmentation_path: str, + segmentation_key: str, + resolution: Tuple[int, int, int], + unit: str, + scale_factors: Tuple[Tuple[int, int, int]], + chunks: Tuple[int, int, int], + skip_existing: bool = True, +): + # Check if we have converted this data already. + have_source = _source_exists(mobie_project, mobie_dataset, source_name) + if have_source and skip_existing: + print(f"Source {source_name} already exists in {mobie_project}:{mobie_dataset}.") + print("Conversion to mobie will be skipped.") + return + elif have_source: + raise NotImplementedError + + with tempfile.TemporaryDirectory() as tmpdir: + add_segmentation( + input_path=segmentation_path, input_key=segmentation_key, + root=mobie_project, dataset_name=mobie_dataset, + segmentation_name=source_name, + resolution=resolution, scale_factors=scale_factors, + chunks=chunks, file_format="bdv.n5", + tmp_folder=tmpdir + ) diff --git a/flamingo_tools/test_data.py b/flamingo_tools/test_data.py index 96d166a..5e52cab 100644 --- a/flamingo_tools/test_data.py +++ b/flamingo_tools/test_data.py @@ -7,7 +7,7 @@ # TODO add metadata def create_test_data(root, size=256, n_channels=2, n_tiles=4): channel_folders = [f"channel{chan_id}" for chan_id in range(n_channels)] - file_name_pattern = "volume_R%i_C%i.tif" + file_name_pattern = "volume_R%i_C%i_I0.tif" for chan_id, channel_folder in enumerate(channel_folders): out_folder = os.path.join(root, channel_folder) os.makedirs(out_folder, exist_ok=True) diff --git a/flamingo_tools/version.py b/flamingo_tools/version.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/flamingo_tools/version.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/scripts/README.md b/scripts/README.md index 535337d..68d3cc3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,18 +1,41 @@ # Segmentation for large lightsheet volumes + +## Installation + +Needs [torch-em](https://github.com/constantinpape/torch-em) in the python environment. See [here](https://github.com/constantinpape/torch-em?tab=readme-ov-file#installation) for installation instructions. (If possible use `mamba` instead of `conda`.) +After setting up the environment you also have to add support for the MoBIE python library via +``` +conda install -c conda-forge mobie_utils +``` + + ## Training Contains the scripts for training a U-Net that predicts foreground probabilties and normalized object distances. + ## Prediction -Contains the scripts for running segmentation for a large volume with a distance prediction U-Net. (Other scripts are work in progress.) +Contains the scripts for running segmentation for a large volume with a distance prediction U-Net, postprocessing the segmentation +and exporting the segmentation result to MoBIE -You can run it like this for input that is stored in n5: +To run the full segmentation workflow, including the export to MoBIE you can use the `segmentation_workflow.py` script as follows: +``` +python segmentation_workflow.py -i /path/to/volume.xml -o /path/to/output_folder --scale 0 -m data_name --model /path/to/model.pt +``` + +Here, `-i` must point to the xml file of the fused data exported from BigSticher, `-o` indicates the output folder where the MoBIE project with the semgentation result will be saved, `--scale` indicates the scale to use for the segmentation, `-m` the name of the data in MoBIE and `--model` the path to the segmentation model. + +### Individual Workflow Steps + +You can also run individual steps of the workflow, like prediction and segmentation: + +You can run it like this for an input volume that is stored in n5, e.g. the fused export from bigstitcher: ``` python run_prediction_distance_unet.py -i /path/to/volume.n5 -k setup0/timepoint0/s0 -m /path/to/model -o /path/to/output_folder ``` -Here, `-i` specifies the input filepath, `-o` the folder where the results are saved and `-k` the internal path for a zarr or n5 file. +Here, `-i` specifies the input filepath, `-o` the folder where the results are saved and `-k` the internal path in the n5 file. The `-m` argument specifies the model to use for prediction. You need to give the path to the folder that contains the checkpoint (the `best.pt` file). You can also run the script for a tif file. In this case you don't need the `-k` parameter: @@ -31,8 +54,4 @@ to downsample the input by a factor of 2. Note that the segmentation result will In addition, the script `postprocess_seg.py` can be used to filter out false positive nucleus segmentations from regions in the segmentation with a low density of segmented nuclei. -You can use the script `to_tif.py` to convert the zarr object to a tif volume for easier viewing (won't work for very large volumes!). - -## Installation - -Needs [torch-em](https://github.com/constantinpape/torch-em) in the python environment. See [here](https://github.com/constantinpape/torch-em?tab=readme-ov-file#installation) for installation instructions. (If possible use `mamba` instead of `conda`.) +You can use the script `to_tif.py` to convert the zarr object to a tif volume for easier viewing (won't work for large volumes!). diff --git a/scripts/data_transfer/README.md b/scripts/data_transfer/README.md index d2a89c4..67f44b6 100644 --- a/scripts/data_transfer/README.md +++ b/scripts/data_transfer/README.md @@ -5,37 +5,30 @@ Current approach to the data transfer: - Log in to SCC login node: $ -- Go to `/scratch1/projects/cca/data/moser` +- Go to "/scratch1/projects/cca/data/moser" - Create subfolder for cochlea to be copied -- Log in via -``` -$ smbclient \\\\wfs-medizin.top.gwdg.de\\ukon-all\$\\ukon100 -U GWDG\\pape41" -``` +- Log in via $ smbclient \\\\wfs-medizin.top.gwdg.de\\ukon-all\$\\ukon100 -U GWDG\\pape41" - Go to the folder with the cochlea to copy (cd works) - Copy the folder via: - recurse ON - prompt OFF - mget * - Copy this to HLRN by logging into it and running -``` + $ rsync pape41:/scratch1/projects/cca/data/moser/ $ rsync -e "ssh -i ~/.ssh/id_rsa_hlrn" -avz pape41@login-mdc.hpc.gwdg.de:/scratch1/projects/cca/data/mose -r/ /mnt/lustre-emmy-hdd/projects/nim00007/data/moser/lightsheet/volumes/ -``` +r/ /mnt/lustre-grete/usr/u12086/moser/lightsheet/ - Remove on SCC ## Next files - UKON100\archiv\imaging\Lightsheet\Huiskengroup_CTLSM\2024\M171_2R_converted_n5 - - unclear what the converted data is -- UKON100\archiv\imaging\Lightsheet\Huiskengroup_CTLSM\2024\155_1L_converted_n5\BDVexport.n5 - - Copied to SCC, need to rsync. +- UKON100\archiv\imaging\Lightsheet\Huiskengroup_CTLSM\2024\155_1L_converted_n5 - UKON100\archiv\imaging\Lightsheet\Huiskengroup_CTLSM\2024\MLR151_2R_converted_n5 - UKON100\archiv\imaging\Lightsheet\Huiskengroup_CTLSM\2024\G11_1L_converted_n5 ## Improvements Try to automate via https://github.com/jborean93/smbprotocol see `sync_smb.py` for ChatGPT's inital version. -Connection not possible from HLRN. ## Transfer Back diff --git a/scripts/prediction/segmentation_workflow.py b/scripts/prediction/segmentation_workflow.py new file mode 100644 index 0000000..89ffb04 --- /dev/null +++ b/scripts/prediction/segmentation_workflow.py @@ -0,0 +1,152 @@ +import argparse +import os +from shutil import rmtree + +import pybdv.metadata as bdv_metadata +import torch +import z5py + +from flamingo_tools.segmentation import run_unet_prediction, filter_isolated_objects +from flamingo_tools.mobie import add_raw_to_mobie, add_segmentation_to_mobie + +MOBIE_ROOT = "/mnt/lustre-emmy-hdd/projects/nim00007/data/moser/lightsheet/mobie" + + +def postprocess_seg(output_folder): + print("Run segmentation postprocessing ...") + seg_path = os.path.join(output_folder, "segmentation.zarr") + seg_key = "segmentation" + + with z5py.File(seg_path, "r") as f: + segmentation = f[seg_key][:] + + seg_filtered, n_pre, n_post = filter_isolated_objects(segmentation) + + with z5py.File(seg_path, "a") as f: + chunks = f[seg_key].chunks + f.create_dataset( + "segmentation_postprocessed", data=seg_filtered, compression="gzip", + chunks=chunks, dtype=seg_filtered.dtype + ) + + +def export_to_mobie(xml_path, segmentation_folder, scale, mobie_dataset, chunks): + # Add to mobie: + + # - raw data (if not yet present) + add_raw_to_mobie( + mobie_project=MOBIE_ROOT, + mobie_dataset=mobie_dataset, + source_name="pv-channel", + xml_path=xml_path, + setup_id=0, + ) + + # TODO enable passing extra channel names + # - additional channels + setup_ids = bdv_metadata.get_setup_ids(xml_path) + if len(setup_ids) > 1: + extra_channel_names = ["gfp_channel", "myo_channel"] + for i, setup_id in enumerate(setup_ids[1:]): + add_raw_to_mobie( + mobie_project=MOBIE_ROOT, + mobie_dataset=mobie_dataset, + source_name=extra_channel_names[i], + xml_path=xml_path, + setup_id=setup_id + ) + + # - segmentation and post-processed segmentation + seg_path = os.path.join(segmentation_folder, "segmentation.zarr") + seg_resolution = bdv_metadata.get_resolution(xml_path, setup_id=0) + if scale == 1: + seg_resolution = [2 * res for res in seg_resolution] + unit = bdv_metadata.get_unit(xml_path, setup_id=0) + + seg_key = "segmentation" + seg_name = "nuclei_fullscale" if scale == 0 else "nuclei_downscaled" + add_segmentation_to_mobie( + mobie_project=MOBIE_ROOT, + mobie_dataset=mobie_dataset, + source_name=seg_name, + segmentation_path=seg_path, + segmentation_key=seg_key, + resolution=seg_resolution, + unit=unit, + scale_factors=4*[[2, 2, 2]], + chunks=chunks, + ) + + seg_key = "segmentation_postprocessed" + seg_name += "_postprocessed" + add_segmentation_to_mobie( + mobie_project=MOBIE_ROOT, + mobie_dataset=mobie_dataset, + source_name=seg_name, + segmentation_path=seg_path, + segmentation_key=seg_key, + resolution=seg_resolution, + unit=unit, + scale_factors=4*[[2, 2, 2]], + chunks=chunks, + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input", required=True) + parser.add_argument("-o", "--output_folder", required=True) + parser.add_argument("-s", "--scale", required=True, type=int) + parser.add_argument("-m", "--mobie_dataset", required=True) + parser.add_argument("--model") + + args = parser.parse_args() + + scale = args.scale + if scale == 0: + min_size = 1000 + elif scale == 1: + min_size = 250 + else: + raise ValueError + + xml_path = args.input + assert os.path.splitext(xml_path)[1] == ".xml" + input_path = bdv_metadata.get_data_path(xml_path, return_absolute_path=True) + + # TODO need to make sure that PV is always setup 0 + input_key = f"setup0/timepoint0/s{scale}" + + have_cuda = torch.cuda.is_available() + chunks = z5py.File(input_path, "r")[input_key].chunks + block_shape = tuple([2 * ch for ch in chunks]) if have_cuda else tuple(chunks) + halo = (16, 64, 64) if have_cuda else (8, 32, 32) + + if args.model is not None: + model = args.model + else: + if scale == 0: + model = "../training/checkpoints/cochlea_distance_unet" + else: + model = "../training/checkpoints/cochlea_distance_unet-train-downsampled" + + run_unet_prediction( + input_path, input_key, args.output_folder, model, + scale=None, min_size=min_size, + block_shape=block_shape, halo=halo, + ) + + postprocess_seg(args.output_folder) + + export_to_mobie(xml_path, args.output_folder, scale, args.mobie_dataset, chunks) + + # clean up: remove segmentation folders + print("Cleaning up intermediate segmentation results") + print("This may take a while, but everything else is done.") + print("You can check the results in the MoBIE project already at:") + print(f"{MOBIE_ROOT}:{args.mobie_dataset}") + rmtree(args.output_folder) + + +if __name__ == "__main__": + main() diff --git a/scripts/prediction/upload_to_s3.py b/scripts/prediction/upload_to_s3.py new file mode 100644 index 0000000..061526a --- /dev/null +++ b/scripts/prediction/upload_to_s3.py @@ -0,0 +1,74 @@ +import os + +import s3fs + +from mobie.metadata import add_remote_project_metadata +from tqdm import tqdm + +# Using incucyte s3 as a temporary measure. +MOBIE_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/moser/lightsheet/mobie" +SERVICE_ENDPOINT = "https://s3.gwdg.de/" +BUCKET_NAME = "incucyte-general/lightsheet" + +# For MoBIE: +# https://s3.gwdg.de/incucyte-general/lightsheet + + +def read_s3_credentials(credential_file): + key, secret = None, None + with open(credential_file) as f: + for line in f: + if line.startswith("aws_access_key_id"): + key = line.rstrip("\n").strip().split(" ")[-1] + if line.startswith("aws_secret_access_key"): + secret = line.rstrip("\n").strip().split(" ")[-1] + if key is None or secret is None: + raise ValueError(f"Invalid credential file {credential_file}") + return key, secret + + +def create_s3_target(url, anon=False, credential_file=None): + client_kwargs = {"endpoint_url": url} + if credential_file is not None: + key, secret = read_s3_credentials(credential_file) + fs = s3fs.S3FileSystem(key=key, secret=secret, client_kwargs=client_kwargs) + else: + fs = s3fs.S3FileSystem(anon=anon, client_kwargs=client_kwargs) + return fs + + +def remote_metadata(): + add_remote_project_metadata(MOBIE_FOLDER, BUCKET_NAME, SERVICE_ENDPOINT) + + +def upload_data(): + target = create_s3_target( + SERVICE_ENDPOINT, + credential_file="./credentials.incucyte" + ) + to_upload = [] + for root, dirs, files in os.walk(MOBIE_FOLDER): + dirs.sort() + for ff in files: + if ff.endswith(".xml"): + to_upload.append(os.path.join(root, ff)) + + print("Uploading", len(to_upload), "files to") + + for path in tqdm(to_upload): + rel_path = os.path.relpath(path, MOBIE_FOLDER) + target.put( + path, os.path.join(BUCKET_NAME, rel_path) + ) + + +# FIXME: access via s3 is not working due to permission issues. +# Maybe this is not working due to bdv fileformat?! +# Make an issue in MoBIE. +def main(): + # remote_metadata() + upload_data() + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fc9a720 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +import runpy +from setuptools import setup, find_packages + +version = runpy.run_path('flamingo_tools/version.py')['__version__'] +setup(name='flamingo_tools', + packages=find_packages(exclude=['test']), + version=version, + author='Constantin Pape', + license='MIT') diff --git a/test/test_data_conversion.py b/test/test_data_conversion.py index 12826f9..1ee7400 100644 --- a/test/test_data_conversion.py +++ b/test/test_data_conversion.py @@ -22,16 +22,7 @@ def test_convert_lightsheet_to_bdv(self): from flamingo_tools import convert_lightsheet_to_bdv out_path = os.path.join(self.folder, "converted_data.n5") - - channel_folders = { - "channel0": "channel0", - "channel1": "channel1", - } - convert_lightsheet_to_bdv( - self.folder, channel_folders, - image_file_name_pattern="volume_R*.tif", - out_path=out_path - ) + convert_lightsheet_to_bdv(self.folder, out_path=out_path) self.assertTrue(os.path.exists(out_path)) xml_path = out_path.replace(".n5", ".xml")