Skip to content

Commit a33f265

Browse files
constantinpapeconstantinpape
committed
Update processing and block extraction scripts
1 parent fd17888 commit a33f265

File tree

15 files changed

+228
-63
lines changed

15 files changed

+228
-63
lines changed

‎flamingo_tools/extract_block_util.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import os
2-
from typing import Optional, List
2+
from typing import Optional, List, Union, Tuple
33

44
import imageio.v3 as imageio
55
import numpy as np
@@ -15,7 +15,7 @@ def extract_block(
1515
output_dir: Optional[str] = None,
1616
input_key: Optional[str] = None,
1717
output_key: Optional[str] = None,
18-
resolution: float = 0.38,
18+
resolution: Union[float, Tuple[float, float, float]] = 0.38,
1919
roi_halo: List[int] = [128, 128, 64],
2020
tif: bool = False,
2121
s3: Optional[bool] = False,
@@ -39,8 +39,7 @@ def extract_block(
3939
s3_service_endpoint: S3 service endpoint.
4040
s3_credentials: File path to credentials for S3 bucket.
4141
"""
42-
coords = [int(round(c)) for c in coords]
43-
coord_string = "-".join([str(c).zfill(4) for c in coords])
42+
coord_string = "-".join([str(int(round(c))).zfill(4) for c in coords])
4443

4544
# Dimensions are inversed to view in MoBIE (x y z) -> (z y x)
4645
# Make sure the coords / roi_halo are not modified in-place.
@@ -77,15 +76,20 @@ def extract_block(
7776
output_key = "raw" if output_key is None else output_key
7877
output_file = os.path.join(output_dir, basename + "_crop_" + coord_string + ".n5")
7978

80-
coords = np.array(coords)
79+
coords = np.array(coords).astype("float")
80+
if not isinstance(resolution, float):
81+
assert len(resolution) == 3
82+
resolution = np.array(resolution)[::-1]
8183
coords = coords / resolution
8284
coords = np.round(coords).astype(np.int32)
8385

8486
roi = tuple(slice(co - rh, co + rh) for co, rh in zip(coords, roi_halo))
8587

8688
if s3:
87-
input_path, fs = s3_utils.get_s3_path(input_path, bucket_name=s3_bucket_name,
88-
service_endpoint=s3_service_endpoint, credential_file=s3_credentials)
89+
input_path, fs = s3_utils.get_s3_path(
90+
input_path, bucket_name=s3_bucket_name,
91+
service_endpoint=s3_service_endpoint, credential_file=s3_credentials
92+
)
8993

9094
data_ = read_image_data(input_path, input_key)
9195
data_roi = data_[roi]

‎reproducibility/label_components/IHC_gerbil.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[
2+
{
3+
"cochlea": "G_EK_000233_L",
4+
"image_channel": "VGlut3",
5+
"cell_type": "ihc",
6+
"unet_version": "v5"
7+
}
8+
]

‎reproducibility/label_components/IHC_v4c_fig2.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,11 @@
2222
"image_channel": "VGlut3",
2323
"cell_type": "ihc",
2424
"unet_version": "v4c"
25+
},
26+
{
27+
"cochlea": "G_EK_000233_L",
28+
"image_channel": "VGlut3",
29+
"cell_type": "ihc",
30+
"unet_version": "v5"
2531
}
2632
]

‎reproducibility/templates_processing/REAMDE.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,17 @@ For IHC segmentation run:
1212
- apply_unet_IHC_template.sbatch
1313
- segment_unet_IHC_template.sbatch
1414

15+
After this, run the following to add segmentation to MoBIE, create component labelings and upload to S3:
16+
- templates_transfer/mobie_segmentatio.sbatch
17+
- templates_transfer/s3_seg_template.sh
18+
- repro_label_components.py
19+
- templates_transfer/s3_seg_template.sh
20+
1521
For ribbon synapse detection without associated IHC segmentation run
1622
- detect_synapse_template.sbatch
1723
For ribbon synapse detection with associated IHC segmentation run
1824
- detect_synapse_marker_template.sbatch
25+
26+
After this, run the following to add detections to MoBIE and upload to S3:
27+
- templates_transfer/mobie_spots_template.sbatch
28+
- s3_synapse_template.sh

‎reproducibility/templates_processing/detect_synapse_marker_template.sbatch‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,21 @@
77
#SBATCH -p grete:shared # the partition
88
#SBATCH -G A100:1 # For requesting 1 A100 GPU.
99
#SBATCH -A nim00007
10-
#SBATCH -c 2
11-
#SBATCH --mem 36G
10+
#SBATCH -c 8
11+
#SBATCH --mem 128G
1212

1313
source ~/.bashrc
14-
micromamba activate micro-sam_gpu
14+
# micromamba activate micro-sam_gpu
15+
micromamba activate sam
1516

1617
# Print out some info.
1718
echo "Submitting job with sbatch from directory: ${SLURM_SUBMIT_DIR}"
1819
echo "Home directory: ${HOME}"
1920
echo "Working directory: $PWD"
2021
echo "Current node: ${SLURM_NODELIST}"
2122

22-
# Run the script
23-
#python myprogram.py $SLURM_ARRAY_TASK_ID
24-
25-
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
23+
# SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
24+
SCRIPT_REPO=/user/pape41/u12086/Work/my_projects/flamingo-tools
2625
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
2726

2827
export SCRIPT_DIR=$SCRIPT_REPO/scripts
@@ -46,7 +45,8 @@ export MAX_DISTANCE=8
4645
echo "OUTPUT_FOLDER $OUTPUT_FOLDER"
4746
echo "MODEL $MODEL"
4847

49-
python ~/flamingo-tools/scripts/synapse_marker_detection/marker_detection.py \
48+
SCRIPT="$SCRIPT_DIR"/synapse_marker_detection/marker_detection.py
49+
python $SCRIPT \
5050
--input "$INPUT_PATH" \
5151
--input_key $INPUT_KEY \
5252
--output_folder "$OUTPUT_FOLDER" \

‎reproducibility/templates_processing/mean_std_SGN_template.sbatch‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ STAIN_CHANNEL=$3
2626
# segmentation name, as it appears in MoBIE, e.g. SGN_v2 or Calb1_SGN_v2
2727
SEG_NAME=$4
2828

29+
# TODO update model to
30+
# /mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/trained_models/SGN/v2_domain_adaptation_mlr99l
31+
# and download the PV channel again
32+
2933
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
34+
3035
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
3136
export SEG_CLASS="sgn"
3237
export INPUT_KEY="setup$STAIN_CHANNEL/timepoint0/s0"

‎reproducibility/templates_transfer/mobie_segmentation_template.sbatch‎

100644100755
Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
#SBATCH --job-name=mobie_segm
3-
#SBATCH -t 01:00:00 # estimated time, adapt to your needs
3+
#SBATCH -t 02:00:00 # estimated time, adapt to your needs
44
#SBATCH [email protected] # change this to your mailaddress
55
#SBATCH --mail-type=FAIL # send mail when job begins and ends
66

@@ -10,27 +10,31 @@
1010
#SBATCH --mem 180G
1111

1212
source ~/.bashrc
13-
source ~/miniconda3/bin/activate
14-
source activate mobie
13+
# source activate mobie
14+
micromamba activate sam
1515

1616
# Run the script
1717

1818
# name of cochlea, as it appears in MoBIE and the NHR
1919
COCHLEA=$1
20-
# data in n5 format, e.g. GEK11L_PV_GFP_01_fused.n5
20+
# data in n5 format, e.g. G_EK_000233_L/IHC_v5/segmentation.zarr
2121
DATA=$2
22-
# segmentation name, as it appears in MoBIE, e.g. PV or Calb1
22+
# segmentation name, as it appears in MoBIE, e.g. SGN_V2
2323
CHANNEL_NAME=$3
2424

2525
MOBIE_PROJECT="/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/mobie_project/cochlea-lightsheet"
26-
INPUT_PATH=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
26+
# /mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/G_EK_000233_L/IHC_v5/segmentation.zarr/
27+
INPUT_PATH=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$DATA"
2728

2829
SEGMENTATION_KEY="segmentation"
2930

3031
RESOLUTION="[0.38,0.38,0.38]"
3132
SCALE_FACTORS="[[2,2,2],[2,2,2],[2,2,2],[2,2,2],[2,2,2],[2,2,2]]"
3233
CHUNKS="[64,64,64]"
3334

34-
mobie.add_segmentation --input_path "$SEGMENTATION_PATH" --input_key "$SEGMENTATION_KEY" --root "$MOBIE_PROJECT" \
35+
echo $INPUT_PATH
36+
echo $MOBIE_PROJECT
37+
38+
mobie.add_segmentation --input_path "$INPUT_PATH" --input_key "$SEGMENTATION_KEY" --root "$MOBIE_PROJECT" \
3539
--dataset_name "$COCHLEA" --name "$CHANNEL_NAME" --resolution "$RESOLUTION" \
3640
--scale_factors "$SCALE_FACTORS" --chunks "$CHUNKS"

‎reproducibility/templates_transfer/mobie_spots_template.sbatch‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,22 @@
1010
#SBATCH --mem 16G
1111

1212
source ~/.bashrc
13-
source ~/miniconda3/bin/activate
14-
source activate mobie
13+
# source ~/miniconda3/bin/activate
14+
# source activate mobie
15+
micromamba activate sam
1516

1617
# Run the script
1718

1819
# name of cochlea, as it appears in MoBIE and the NHR
1920
COCHLEA=$1
21+
2022
# segmentation name, as it appears in MoBIE, e.g. synapses_v3 or synapses_v3_ihc_v4
2123
SPOT_NAME=$2
2224

2325
MOBIE_PROJECT="/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/mobie_project/cochlea-lightsheet"
2426

25-
TABLE_PATH=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SPOT_NAME"/synapse_detection.tsv # synapse_detection_filtered.tsv
27+
# TABLE_PATH=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SPOT_NAME"/synapse_detection.tsv
28+
TABLE_PATH=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SPOT_NAME"/synapse_detection_filtered.tsv
2629

2730
mobie.add_spots --input_table "$TABLE_PATH" --root "$MOBIE_PROJECT" \
2831
--dataset_name "$COCHLEA" --name "$SPOT_NAME"

‎reproducibility/templates_transfer/s3_cochlea_template.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ export SERVICE_ENDPOINT="https://s3.fs.gwdg.de"
99
mobie.add_remote_metadata -i $MOBIE_DIR -s $SERVICE_ENDPOINT -b $BUCKET_NAME
1010

1111
rclone --progress copyto "$MOBIE_DIR"/"$COCHLEA" cochlea-lightsheet:cochlea-lightsheet/"$COCHLEA"
12+
rclone --progress copyto "$MOBIE_DIR"/project.json cochlea-lightsheet:cochlea-lightsheet/project.json

‎reproducibility/templates_transfer/s3_seg_template.sh‎

100644100755
File mode changed.

0 commit comments

Comments
 (0)