Skip to content

Commit b00918b

Browse files
schilling40schilling40
committed
Scripts for data processing and transfer
1 parent efeaf4c commit b00918b

16 files changed

+531
-13
lines changed

‎reproducibility/templates_processing/apply_unet_IHC_template.sbatch‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=apply-unet-IHC
3+
#SBATCH -t 02:00:00 # for gerbil up to ~3 hours
4+
5+
#SBATCH -p grete:shared # the partition
6+
#SBATCH -G A100:1 # For requesting 1 A100 GPU.
7+
#SBATCH -c 1
8+
#SBATCH --mem 24G
9+
#SBATCH -a 0-9
10+
11+
source ~/.bashrc
12+
micromamba activate micro-sam_gpu
13+
14+
# Print out some info.
15+
echo "Submitting job with sbatch from directory: ${SLURM_SUBMIT_DIR}"
16+
echo "Home directory: ${HOME}"
17+
echo "Working directory: $PWD"
18+
echo "Current node: ${SLURM_NODELIST}"
19+
20+
# Run the script
21+
22+
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
23+
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
24+
25+
export SCRIPT_DIR=$SCRIPT_REPO/scripts
26+
27+
# name of cochlea, as it appears in MoBIE and the NHR
28+
COCHLEA=$1
29+
# data in n5 format, e.g. GEK11L_PV_GFP_01_fused.n5
30+
DATA=$2
31+
# channel in n5 folder, e.g. 0, 1, or 2
32+
STAIN_CHANNEL=$3
33+
# segmentation name, as it appears in MoBIE, e.g. IHC_v4
34+
SEG_NAME=$4
35+
36+
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
37+
38+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
39+
40+
export MODEL=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/trained_models/IHC/v4_cochlea_distance_unet_IHC_supervised_2025-07-14
41+
export PREDICTION_INSTANCES=10
42+
export INPUT_KEY="setup$STAIN_CHANNEL/timepoint0/s0"
43+
44+
echo "Input directory: ${INPUT}"
45+
echo "Output directory: ${OUTPUT_FOLDER}"
46+
echo "Model: ${MODEL}"
47+
48+
cmd_array=( 'import sys,os;'
49+
'sys.path.insert(0,os.environ["SCRIPT_DIR"]);'
50+
'import unet_prediction;'
51+
'unet_prediction.run_unet_prediction_slurm(input_path=os.environ["INPUT"],'
52+
'output_folder=os.environ["OUTPUT_FOLDER"],model_path=os.environ["MODEL"],'
53+
'input_key=os.environ["INPUT_KEY"],'
54+
'prediction_instances=os.environ["PREDICTION_INSTANCES"])')
55+
cmd="${cmd_array[*]}"
56+
python -c "$cmd"
57+

‎reproducibility/segmentation_pipeline/apply_unet_SGN_template.sbatch‎ renamed to ‎reproducibility/templates_processing/apply_unet_SGN_template.sbatch‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
#SBATCH --job-name=apply-unet-SGN
3-
#SBATCH -t 01:00:00 # estimated time, adapt to your needs
3+
#SBATCH -t 02:00:00 # for gerbil up to ~3 hours
44

55
#SBATCH -p grete:shared # the partition
66
#SBATCH -G A100:1 # For requesting 1 A100 GPU.
@@ -24,15 +24,22 @@ cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
2424

2525
export SCRIPT_DIR=$SCRIPT_REPO/scripts
2626

27+
# name of cochlea, as it appears in MoBIE and the NHR
2728
COCHLEA=$1
29+
# data in n5 format, e.g. GEK11L_PV_GFP_01_fused.n5
30+
DATA=$2
31+
# channel in n5 folder, e.g. 0, 1, or 2
32+
STAIN_CHANNEL=$3
33+
# segmentation name, as it appears in MoBIE, e.g. SGN_v2 or Calb1_SGN_v2
34+
SEG_NAME=$4
2835

29-
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/MLR214L_Calb1_PV_CR_012_fused.n5
36+
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
3037

31-
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/SGN_v2
38+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
3239

33-
export MODEL=/mnt/lustre-grete/usr/u15000/checkpoints/cochlea_distance_unet_SGN_supervised_2025-05-27/
40+
export MODEL=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/trained_models/SGN/v2_cochlea_distance_unet_SGN_supervised_2025-05-27
3441
export PREDICTION_INSTANCES=10
35-
export INPUT_KEY="setup1/timepoint0/s0"
42+
export INPUT_KEY="setup$STAIN_CHANNEL/timepoint0/s0"
3643

3744
echo "Input directory: ${INPUT}"
3845
echo "Output directory: ${OUTPUT_FOLDER}"

‎reproducibility/templates_processing/detect_synapse_marker_template.sbatch‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=synapse-marker
3+
#SBATCH -t 08:00:00 # estimated time, adapt to your needs
4+
#SBATCH [email protected] # change this to your mailaddress
5+
#SBATCH --mail-type=FAIL # send mail when job begins and ends
6+
7+
#SBATCH -p grete:shared # the partition
8+
#SBATCH -G A100:1 # For requesting 1 A100 GPU.
9+
#SBATCH -A nim00007
10+
#SBATCH -c 2
11+
#SBATCH --mem 36G
12+
13+
source ~/.bashrc
14+
micromamba activate micro-sam_gpu
15+
16+
# Print out some info.
17+
echo "Submitting job with sbatch from directory: ${SLURM_SUBMIT_DIR}"
18+
echo "Home directory: ${HOME}"
19+
echo "Working directory: $PWD"
20+
echo "Current node: ${SLURM_NODELIST}"
21+
22+
# Run the script
23+
#python myprogram.py $SLURM_ARRAY_TASK_ID
24+
25+
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
26+
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
27+
28+
export SCRIPT_DIR=$SCRIPT_REPO/scripts
29+
30+
# name of cochlea, as it appears in MoBIE and the NHR
31+
COCHLEA=$1
32+
# image channel, e.g. CTBP2 or RibA
33+
IMAGE_CHANNEL=$2
34+
# segmentation name, as it appears in MoBIE, e.g. synapses_v3
35+
IHC_SEG=$3
36+
37+
export INPUT_PATH="$COCHLEA"/images/ome-zarr/"$IMAGE_CHANNEL".ome.zarr
38+
export MASK_PATH="$COCHLEA"/images/ome-zarr/"$IHC_SEG".ome.zarr
39+
40+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/synapses_v3_"$IHC_SEG"
41+
42+
export MODEL=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/trained_models/Synapses/synapse_detection_model_v3.pt
43+
export INPUT_KEY="s0"
44+
export MAX_DISTANCE=8
45+
46+
echo "OUTPUT_FOLDER $OUTPUT_FOLDER"
47+
echo "MODEL $MODEL"
48+
49+
python ~/flamingo-tools/scripts/synapse_marker_detection/marker_detection.py \
50+
--input "$INPUT_PATH" \
51+
--input_key $INPUT_KEY \
52+
--output_folder "$OUTPUT_FOLDER" \
53+
--mask "$MASK_PATH" \
54+
--model $MODEL \
55+
--max_distance $MAX_DISTANCE \
56+
--s3
57+

‎reproducibility/templates_processing/detect_synapse_template.sbatch‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=synapse-detect
3+
#SBATCH -t 42:00:00 # estimated time, adapt to your needs
4+
#SBATCH [email protected] # change this to your mailaddress
5+
#SBATCH --mail-type=FAIL # send mail when job begins and ends
6+
7+
#SBATCH -p grete:shared # the partition
8+
#SBATCH -G A100:1 # For requesting 1 A100 GPU.
9+
#SBATCH -A nim00007
10+
#SBATCH -c 2
11+
#SBATCH --mem 500G
12+
13+
source ~/.bashrc
14+
micromamba activate micro-sam_gpu
15+
16+
# Print out some info.
17+
echo "Submitting job with sbatch from directory: ${SLURM_SUBMIT_DIR}"
18+
echo "Home directory: ${HOME}"
19+
echo "Working directory: $PWD"
20+
echo "Current node: ${SLURM_NODELIST}"
21+
22+
# Run the script
23+
#python myprogram.py $SLURM_ARRAY_TASK_ID
24+
25+
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
26+
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
27+
28+
export SCRIPT_DIR=$SCRIPT_REPO/scripts
29+
30+
# name of cochlea, as it appears in MoBIE and the NHR
31+
COCHLEA=$1
32+
# image channel, e.g. CTBP2 or RibA
33+
IMAGE_CHANNEL=$2
34+
# segmentation name, as it appears in MoBIE, e.g. synapses_v3
35+
IHC_SEG=$3
36+
37+
export INPUT_PATH="$COCHLEA"/images/ome-zarr/"$IMAGE_CHANNEL".ome.zarr
38+
export MASK_PATH="$COCHLEA"/images/ome-zarr/"$IHC_SEG".ome.zarr
39+
40+
# data on NHR
41+
# export INPUT_PATH=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
42+
# export INPUT_KEY="setup$STAIN_CHANNEL/timepoint0/s0"
43+
44+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/synapses_v3
45+
46+
if ! [[ -f $OUTPUT_FOLDER ]] ; then
47+
mkdir -p "$OUTPUT_FOLDER"
48+
fi
49+
50+
export MODEL=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/trained_models/Synapses/synapse_detection_model_v3.pt
51+
52+
echo "OUTPUT_FOLDER $OUTPUT_FOLDER"
53+
echo "MODEL $MODEL"
54+
55+
python ~/flamingo-tools/scripts/synapse_marker_detection/run_prediction.py \
56+
--input "$INPUT_PATH" \
57+
--input_key "$INPUT_KEY" \
58+
--output_folder "$OUTPUT_FOLDER" \
59+
--model $MODEL \
60+
--s3
61+

‎reproducibility/templates_processing/mean_std_IHC_template.sbatch‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=mean-std-IHC
3+
#SBATCH -t 01:00:00 # usually takes ~20 min
4+
5+
#SBATCH -p standard96s:shared # the partition
6+
#SBATCH -A nim00007
7+
#SBATCH -c 3
8+
#SBATCH --mem 128G
9+
10+
source ~/.bashrc
11+
micromamba activate flamingo13
12+
13+
# Run the script
14+
15+
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
16+
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
17+
18+
export SCRIPT_DIR=$SCRIPT_REPO/scripts
19+
20+
# name of cochlea, as it appears in MoBIE and the NHR
21+
COCHLEA=$1
22+
# data in n5 format, e.g. GEK11L_PV_GFP_01_fused.n5
23+
DATA=$2
24+
# channel in n5 folder, e.g. 0, 1, or 2
25+
STAIN_CHANNEL=$3
26+
# segmentation name, as it appears in MoBIE, e.g. IHC_v4
27+
SEG_NAME=$4
28+
29+
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
30+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
31+
export SEG_CLASS="ihc"
32+
export INPUT_KEY="setup$STAIN_CHANNEL/timepoint0/s0"
33+
34+
if ! [[ -f $OUTPUT_FOLDER ]] ; then
35+
mkdir -p "$OUTPUT_FOLDER"
36+
fi
37+
38+
echo "Input directory: ${INPUT}"
39+
echo "Output directory: ${OUTPUT_FOLDER}"
40+
41+
cmd_array=( 'import sys,os;'
42+
'sys.path.insert(0,os.environ["SCRIPT_DIR"]);'
43+
'import unet_prediction;'
44+
'unet_prediction.run_unet_prediction_preprocess_slurm(input_path=os.environ["INPUT"],'
45+
'input_key=os.environ["INPUT_KEY"],'
46+
'output_folder=os.environ["OUTPUT_FOLDER"],seg_class=os.environ["SEG_CLASS"])')
47+
cmd="${cmd_array[*]}"
48+
python -c "$cmd"
49+

‎reproducibility/segmentation_pipeline/mean_std_SGN_template.sbatch‎ renamed to ‎reproducibility/templates_processing/mean_std_SGN_template.sbatch‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
#SBATCH --job-name=mean-std-SGN
3-
#SBATCH -t 02:00:00 # estimated time, adapt to your needs
3+
#SBATCH -t 01:00:00 # usually takes ~20 min
44

55
#SBATCH -p standard96s:shared # the partition
66
#SBATCH -A nim00007
@@ -17,12 +17,19 @@ cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
1717

1818
export SCRIPT_DIR=$SCRIPT_REPO/scripts
1919

20+
# name of cochlea, as it appears in MoBIE and the NHR
2021
COCHLEA=$1
21-
22-
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/GEK11L_PV_GFP_01_fused.n5
23-
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/SGN_v2
22+
# data in n5 format, e.g. GEK11L_PV_GFP_01_fused.n5
23+
DATA=$2
24+
# channel in n5 folder, e.g. 0, 1, or 2
25+
STAIN_CHANNEL=$3
26+
# segmentation name, as it appears in MoBIE, e.g. SGN_v2 or Calb1_SGN_v2
27+
SEG_NAME=$4
28+
29+
export INPUT=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/"$COCHLEA"/"$DATA"
30+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
2431
export SEG_CLASS="sgn"
25-
export INPUT_KEY="setup0/timepoint0/s0"
32+
export INPUT_KEY="setup$STAIN_CHANNEL/timepoint0/s0"
2633

2734
if ! [[ -f $OUTPUT_FOLDER ]] ; then
2835
mkdir -p "$OUTPUT_FOLDER"

‎reproducibility/templates_processing/segment_unet_IHC_template.sbatch‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=segment-unet-SGN
3+
#SBATCH --time 10:00:00 # for gerbil up to ~30 hours
4+
5+
#SBATCH -p standard96s:shared # the partition
6+
#SBATCH -A nim00007
7+
#SBATCH -c 8
8+
#SBATCH --mem 400G
9+
10+
source ~/.bashrc
11+
micromamba activate micro-sam_gpu
12+
13+
# Print out some info.
14+
echo "Submitting job with sbatch from directory: ${SLURM_SUBMIT_DIR}"
15+
echo "Home directory: ${HOME}"
16+
echo "Working directory: $PWD"
17+
echo "Current node: ${SLURM_NODELIST}"
18+
19+
# Run the script
20+
#python myprogram.py $SLURM_ARRAY_TASK_ID
21+
22+
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
23+
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
24+
25+
# name of cochlea, as it appears in MoBIE and the NHR
26+
COCHLEA=$1
27+
# segmentation name, as it appears in MoBIE, e.g. IHC_v4
28+
SEG_NAME=$2
29+
30+
export SCRIPT_DIR=$SCRIPT_REPO/scripts
31+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
32+
export MIN_SIZE=1000
33+
34+
# v4a
35+
# export CENTER_DISTANCE_THRESHOLD=0.4
36+
# export BOUNDARY_DISTANCE_THRESHOLD=0.5
37+
# export DISTANCE_SMOOTHING=0
38+
39+
# v4b
40+
export CENTER_DISTANCE_THRESHOLD=0.5
41+
export BOUNDARY_DISTANCE_THRESHOLD=0.6
42+
export DISTANCE_SMOOTHING=0.6
43+
44+
echo "Output directory: ${OUTPUT_FOLDER}"
45+
46+
cmd_array=( 'import sys,os;'
47+
'sys.path.insert(0,os.environ["SCRIPT_DIR"]);'
48+
'import unet_prediction;'
49+
'unet_prediction.run_unet_segmentation_slurm(output_folder=os.environ["OUTPUT_FOLDER"],'
50+
'min_size=os.environ["MIN_SIZE"])')
51+
cmd="${cmd_array[*]}"
52+
python -c "$cmd"
53+

‎reproducibility/segmentation_pipeline/segment_unet_SGN_template.sbatch‎ renamed to ‎reproducibility/templates_processing/segment_unet_SGN_template.sbatch‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
2-
#SBATCH --job-name=segment-unet-SGN_M-LR-000214-L
3-
#SBATCH --time 10:00:00 # estimated time, adapt to your needs
2+
#SBATCH --job-name=segment-unet-SGN
3+
#SBATCH --time 10:00:00 # for gerbil up to ~30 hours
44

55
#SBATCH -p standard96s:shared # the partition
66
#SBATCH -A nim00007
@@ -22,10 +22,13 @@ echo "Current node: ${SLURM_NODELIST}"
2222
SCRIPT_REPO=/user/schilling40/u15000/flamingo-tools
2323
cd "$SCRIPT_REPO"/flamingo_tools/segmentation/ || exit
2424

25+
# name of cochlea, as it appears in MoBIE and the NHR
2526
COCHLEA=$1
27+
# segmentation name, as it appears in MoBIE, e.g. SGN_v2 or Calb1_SGN_v2
28+
SEG_NAME=$2
2629

2730
export SCRIPT_DIR=$SCRIPT_REPO/scripts
28-
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/SGN_v2
31+
export OUTPUT_FOLDER=/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/predictions/"$COCHLEA"/"$SEG_NAME"
2932
export MIN_SIZE=1000
3033
export BOUNDARY_DISTANCE_THRESHOLD=0.5
3134

0 commit comments

Comments
 (0)