Skip to content

Commit f7d43e3

Browse files
authored
Merge pull request #14 from ICR-RSE-Group/fix_issue_13
[long-reads]: add customised input option
2 parents 4ee9524 + 862cf77 commit f7d43e3

File tree

8 files changed

+122
-104
lines changed

8 files changed

+122
-104
lines changed

custom_files/pipeline_project_map.json

Lines changed: 0 additions & 5 deletions
This file was deleted.
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
epi2me-human-variation:
2+
nf-long-reads:
3+
is_inputType_path: False
4+
epi2me-somatic-variation:
5+
mopopgen-support:
6+
is_inputType_path: True
7+
nfcore-rnaseq:
8+
nf-tp53:
9+
is_inputType_path: True

pages/run_pipeline.py

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import tabs.tab_command as tt
44
from pipeline_project_map import map_pipeline_project
5-
from shared.sessionstate import retrieve_all_from_ss, save_in_ss, ss_set
5+
from shared.sessionstate import retrieve_all_from_ss, save_in_ss, ss_get, ss_set
66
from shared.visual import header
77

88

@@ -29,35 +29,53 @@ def reset_button_state():
2929
password = ss_values["password"]
3030
PROJECT = ss_values["PROJECT"]
3131
JOB_ID = ss_values["JOB_ID"]
32-
WORKDIR = ss_values["WORK_DIR"]
32+
WORK_DIR = ss_values["WORK_DIR"]
3333
OUTPUT_DIR = ss_values["OUTPUT_DIR"]
3434
run_pipeline_clicked = ss_values["run_pipeline_clicked"]
3535
button_clicked = ss_values["button_clicked"]
36+
custom_sample_list = ss_values["custom_sample_list"] # only availanle if custom sample is selected
3637

37-
samples = ["all", "demo"] # , "customised"]
38+
samples = ["all", "demo", "customised"] # , "test"]
3839

3940
# Create the selectbox and update session state
40-
options = ["select"] + list(map_pipeline_project.keys())
41-
index = options.index(PIPELINE)
42-
PIPELINE = st.selectbox("Select a pipeline", options=options, index=index) # , key="PIPELINE")
41+
pipeline_options = ["select"] + list(map_pipeline_project.keys())
42+
index = pipeline_options.index(PIPELINE)
43+
PIPELINE = st.selectbox("Select a pipeline", options=pipeline_options, index=index) # , key="PIPELINE")
4344
# adding "select" as the first and default choice
4445
if PIPELINE != "select":
46+
project_options = list(map_pipeline_project[PIPELINE].keys())
47+
4548
PROJECT = st.selectbox(
4649
"Select your project",
47-
options=map_pipeline_project[PIPELINE],
48-
index=map_pipeline_project[PIPELINE].index(PIPELINE) if PIPELINE in map_pipeline_project[PIPELINE] else 0,
50+
options=project_options,
51+
index=project_options.index(PIPELINE) if PIPELINE in project_options else 0,
4952
on_change=reset_button_state,
5053
)
51-
54+
# samples here depend on the pipeline: human variation requires bam files, rnaseq requires a samplesheet
5255
SAMPLE = st.selectbox(
5356
"Select your samples",
5457
options=samples,
5558
index=samples.index(SAMPLE) if SAMPLE in samples else 0,
5659
on_change=reset_button_state,
5760
)
5861

59-
WORK_DIR = st.text_input("Working directory", value=SCRATCH)
60-
OUTPUT_DIR = st.text_input("Output directory", value=SCRATCH)
62+
# If "customised" is selected, show additional input
63+
if SAMPLE == "customised":
64+
is_input_type_path = map_pipeline_project[PIPELINE][PROJECT]["is_inputType_path"]
65+
msg = "Enter your sample names (comma-separated)"
66+
if is_input_type_path:
67+
msg = "Enter path to samplesheet"
68+
custom_samples = st.text_input(msg, key="custom_samples", value=",".join(custom_sample_list))
69+
70+
# Optionally process the input into a list
71+
if custom_samples:
72+
custom_sample_list = [s.strip() for s in custom_samples.split(",") if s.strip()]
73+
74+
st.write("Your custom samples:", custom_sample_list)
75+
ss_set("custom_sample_list", custom_sample_list)
76+
77+
WORK_DIR = st.text_input("Working directory", value=WORK_DIR or SCRATCH)
78+
OUTPUT_DIR = st.text_input("Output directory", value=OUTPUT_DIR or SCRATCH)
6179

6280
# passing inputs between tabs
6381
if OK:
@@ -69,6 +87,7 @@ def reset_button_state():
6987
selected_samples=SAMPLE,
7088
work_dir=WORK_DIR,
7189
output_dir=OUTPUT_DIR,
90+
custom_sample_list=custom_sample_list,
7291
)
7392
save_in_ss(
7493
{
@@ -84,11 +103,12 @@ def reset_button_state():
84103
"SAMPLE": SAMPLE,
85104
"PIPELINE": PIPELINE,
86105
"PROJECT": PROJECT,
87-
"JOB_ID": JOB_ID,
88-
"WORKDIR": WORK_DIR,
106+
# "JOB_ID": JOB_ID,
107+
"WORK_DIR": WORK_DIR,
89108
"OUTPUT_DIR": OUTPUT_DIR,
90109
"run_pipeline_clicked": run_pipeline_clicked,
91110
"button_clicked": button_clicked,
111+
"custom_sample_list": custom_sample_list,
92112
}
93113
)
94114
else:

pipeline_project_map.py

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
1-
import json
1+
import yaml
22

3+
file_path = "custom_files/pipeline_project_map_specifications.yaml"
34

4-
def load_json_to_dict(file_path):
5-
with open(file_path, "r") as file:
6-
data = json.load(file)
7-
return data
8-
9-
10-
file_path = "custom_files/pipeline_project_map.json"
11-
map_pipeline_project = load_json_to_dict(file_path)
5+
with open(file_path, "r") as f:
6+
map_pipeline_project = yaml.safe_load(f)

shared/command_helper.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@ def get_path_to_script(selected_pipeline, selected_project, selected="all"):
99
script_mapping = {
1010
"all": "launch_samples.sh",
1111
"demo": "launch_demo.sh",
12-
"single": "launch_sample_analysis.sh", # not yet supported
12+
"customised": "launch_samples.sh",
1313
}
1414

1515
if selected in script_mapping:
1616
return os.path.join(base_path, script_mapping[selected])
1717

18-
raise ValueError(f"Invalid selection '{selected}'. Only 'all' and 'demo' are supported.")
18+
raise ValueError(f"Invalid selection '{selected}'. Only 'customised' and 'demo' are supported.")
1919

2020

2121
# launch command based on the project
@@ -27,6 +27,7 @@ def pipe_cmd(
2727
selected_samples="all",
2828
work_dir="work",
2929
output_dir="output",
30+
custom_sample_list=[],
3031
):
3132
def get_pipeline_command():
3233
"""Generate the pipeline execution command based on the sample selection."""
@@ -42,7 +43,7 @@ def get_pipeline_command():
4243

4344
if selected_samples == "demo":
4445
cmd_pipeline += f"sbatch -o {log_out} -e {log_err} {path_to_script} {work_dir} {output_dir}"
45-
elif selected_samples == "all":
46+
elif selected_samples == "all": # this has no more sense since we have to specify the sample name index
4647
cmd_pipeline += f"sbatch -o {log_out} -e {log_err} {path_to_script} --work-dir {work_dir} --outdir {output_dir}"
4748
##./your_script.sh --env "/my/custom/env" --work-dir "my_work" --outdir "my_output" --config "my_config" --params "parans.json"
4849
# Usage:
@@ -52,7 +53,14 @@ def get_pipeline_command():
5253
# --env "/data/rds/DIT/SCICOM/SCRSE/shared/conda/nextflow_env" \
5354
# --params "/data/params/parameters.json" \
5455
# --config "custom_config.config"
56+
elif selected_samples == "customised":
57+
if not len(custom_sample_list):
58+
print("custom_sample_list cannot be empty")
5559

60+
tab_separated_string = "\t".join(custom_sample_list)
61+
cmd_pipeline += f"sbatch -o {log_out} -e {log_err} {path_to_script} --work-dir {work_dir} --outdir {output_dir} --samples {tab_separated_string}"
62+
# elif selected_samples == "test":
63+
# cmd_pipeline += f"sbatch -o {log_out} -e {log_err} /data/scratch/DCO/DIGOPS/SCIENCOM/msarkis/NF-project-configurations/test.sh --work-dir {work_dir} --outdir {output_dir}"
5664
return cmd_pipeline.strip()
5765

5866
# Command mappings

shared/ss_defaults.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ keys_defaults:
1717
group_selection: "Select an option"
1818
WORK_DIR: None
1919
OUTPUT_DIR: None
20+
custom_sample_list: []

tabs/tab_command.py

Lines changed: 62 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -4,113 +4,102 @@
44

55
import shared.command_helper as cmd_hlp
66
import shared.helpers as hlp
7-
from shared.sessionstate import retrieve_all_from_ss, ss_get, ss_set
87

9-
# pull saved values if set, otherwise set to defaults
10-
ss_values = retrieve_all_from_ss()
11-
OK = ss_values["OK"]
12-
MY_SSH = ss_values["MY_SSH"]
13-
username = ss_values["user_name"]
14-
server = ss_values["server"]
15-
GROUP = ss_values["GROUP"]
16-
GROUPS = ss_values["GROUPS"]
17-
SCRATCH = ss_values["SCRATCH"]
18-
RDS = ss_values["RDS"]
19-
SAMPLE = ss_values["SAMPLE"]
20-
PIPELINE = ss_values["PIPELINE"]
21-
password = ss_values["password"]
22-
PROJECT = ss_values["PROJECT"]
23-
JOB_ID = ss_values["JOB_ID"]
24-
WORKDIR = ss_values["WORK_DIR"]
25-
OUTPUT_DIR = ss_values["OUTPUT_DIR"]
26-
run_pipeline_clicked = ss_values["run_pipeline_clicked"]
278

9+
def tab(
10+
username,
11+
MY_SSH,
12+
selected_pipeline,
13+
selected_project,
14+
selected_samples="all",
15+
work_dir="work",
16+
output_dir="output",
17+
custom_sample_list=[],
18+
):
19+
# --- Initialize session state ---
20+
st.session_state.setdefault("username", username)
21+
st.session_state.setdefault("JOB_ID", "17379785")
22+
st.session_state.setdefault("run_pipeline_clicked", False)
2823

29-
def display_log(title, log_path, output_container):
30-
"""function to display log content."""
31-
try:
32-
with hlp.st_capture(output_container.code):
33-
print(f"{title} log:", log_path)
34-
log_content = MY_SSH.read_file(log_path)
35-
print(log_content)
36-
except Exception as e:
37-
st.error(f"Failed to read {title.lower()} log: {e}")
38-
39-
40-
def tab(username, MY_SSH, selected_pipeline, selected_project, selected_samples="all", work_dir="work", output_dir="output"):
41-
JOB_ID = ss_values["JOB_ID"]
42-
43-
cols = st.columns([1, 1, 1])
24+
# --- Display username input ---
25+
cols = st.columns([1])
4426
with cols[0]:
45-
username = st.text_input(
46-
"Username(s):",
47-
username,
48-
key="username-mod",
49-
help="Enter your username e.g. ralcraft",
50-
)
27+
st.session_state["username"] = st.text_input("Username(s):", st.session_state["username"])
5128

29+
# --- Log display helper ---
30+
def display_log(title, log_path, output_container):
31+
try:
32+
log_file = MY_SSH.read_file(log_path)
33+
log_content = log_file.read() if hasattr(log_file, "read") else str(log_file)
34+
output_container.code(log_content, language="bash")
35+
except Exception as e:
36+
st.error(f"❌ Failed to read {title.lower()} log: {e}")
37+
38+
# --- Run pipeline logic ---
5239
def run_nextflow():
5340
cmd_pipeline = cmd_hlp.pipe_cmd(
54-
username,
41+
st.session_state["username"],
5542
selected_pipeline,
5643
selected_project,
5744
cmd_num=0,
5845
selected_samples=selected_samples,
5946
output_dir=output_dir,
6047
work_dir=work_dir,
48+
custom_sample_list=custom_sample_list,
6149
)
6250
st.code(cmd_pipeline)
63-
_dict = MY_SSH.run_cmd(cmd_pipeline)
64-
# process output to get job id
65-
match = re.search(r"Submitted batch job (\d+)", _dict["output"])
66-
JOB_ID = match.group(1) if match else None
67-
st.success(f"✅ Job ID: {JOB_ID}")
68-
return JOB_ID
69-
# to do, we need to wait for an asynchronous answer regarding slurm?
51+
result = MY_SSH.run_cmd(cmd_pipeline)
52+
match = re.search(r"Submitted batch job (\d+)", result["output"])
53+
job_id = match.group(1) if match else None
54+
st.session_state["JOB_ID"] = job_id
55+
st.success(f"✅ Job submitted. ID: {job_id}")
56+
return job_id
7057

58+
# --- Tabs ---
7159
tabP, tabL, tabQ = st.tabs(["Run pipeline", "Check logs", "Check queues"])
60+
61+
# --- Pipeline tab ---
62+
with tabP:
63+
if st.button("Run the selected pipeline"):
64+
st.session_state["run_pipeline_clicked"] = True
65+
with st.spinner("Submitting pipeline..."):
66+
try:
67+
run_nextflow()
68+
except Exception as e:
69+
st.error(f"Pipeline error: {e}")
70+
71+
if st.session_state["JOB_ID"]:
72+
st.success(f"Running Job ID: {st.session_state['JOB_ID']}")
73+
74+
# --- Logs tab ---
7275
with tabL:
7376
if st.button("Get Logs"):
74-
if JOB_ID == "":
77+
# st.write("📦 session_state:", dict(st.session_state))
78+
job_id = st.session_state.get("JOB_ID")
79+
st.write("📌 Accessed JOB_ID:", job_id) # DEBUG
80+
if not job_id:
7581
st.error("No job was launched yet")
7682
else:
77-
log_out = f"{work_dir}/logs/{JOB_ID}.out"
78-
log_err = f"{work_dir}/logs/{JOB_ID}.err"
83+
log_out = f"{work_dir}/logs/{job_id}.out"
84+
log_err = f"{work_dir}/logs/{job_id}.err"
7985
tO, tE = st.tabs(["Output", "Error"])
8086
outputO, outputE = tO.empty(), tE.empty()
81-
with st.spinner("Fetching...", show_time=True):
87+
with st.spinner("Fetching logs..."):
8288
display_log("Output", log_out, outputO)
8389
display_log("Error", log_err, outputE)
8490

91+
# --- Queues tab ---
8592
with tabQ:
8693
if st.button("Check slurm queues"):
8794
output = st.empty()
88-
with st.spinner("Checking...", show_time=True):
95+
with st.spinner("Checking queue..."):
8996
with hlp.st_capture(output.code):
90-
cmd_pipeline = cmd_hlp.pipe_cmd(username, cmd_num=1)
91-
print("Executing command\n", cmd_pipeline)
97+
cmd_pipeline = cmd_hlp.pipe_cmd(st.session_state["username"], cmd_num=1)
9298
try:
9399
results = MY_SSH.run_cmd(cmd_pipeline)
94-
if results["err"] != None:
100+
if results["err"] is not None:
95101
st.error(results["err"])
96102
else:
97-
print("------------------------------")
98103
print(results["output"])
99104
except Exception as e:
100105
st.error(f"Error {e}")
101-
with tabP:
102-
# disable button once the user clicks a first time. by default it gets disabled after calling the callback
103-
clicked = st.button(f"Run the selected nextflow pipeline for {username}", disabled=ss_get("run_pipeline_clicked"))
104-
JOB_ID = ss_get("JOB_ID")
105-
if JOB_ID is not None:
106-
st.success(f"Running Job ID: {JOB_ID}")
107-
if clicked:
108-
ss_set("run_pipeline_clicked", True)
109-
output = st.empty()
110-
with st.spinner("Starting...", show_time=True):
111-
with hlp.st_capture(output.code):
112-
try:
113-
JOB_ID = run_nextflow()
114-
ss_set("JOB_ID", JOB_ID)
115-
except Exception as e:
116-
st.error(f"Error {e}")

tabs/tab_logon.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ def create_radio(GROUPS):
8080
OK, MY_SSH, msg, GROUPS = handle_login(server, sftp_server, username, password)
8181
if OK:
8282
st.success(msg)
83-
create_radio(GROUPS)
8483
else:
8584
st.error(msg)
8685

@@ -98,6 +97,8 @@ def create_radio(GROUPS):
9897
"GROUP": GROUP,
9998
"GROUPS": GROUPS,
10099
"SCRATCH": SCRATCH,
100+
"WORK_DIR": SCRATCH,
101+
"OUTPUT_DIR": SCRATCH, # set default to scratch
101102
"RDS": RDS,
102103
"SAMPLE": SAMPLE,
103104
"PIPELINE": PIPELINE,

0 commit comments

Comments
 (0)