Skip to content

Commit 270af4f

Browse files
mflynn-lanlmflynn-lanl
authored
Merge pull request #413 from microbiomedata/412-update-bulk-submission-templates
Update templates and handle empty rows and user input error
2 parents 842c427 + abfe74e commit 270af4f

File tree

3 files changed

+31
-20
lines changed

3 files changed

+31
-20
lines changed

‎data/docs/bulk-submissions/NMDC-EDGE-Metagenomics-ReadsQC-bulk-submission.xlsx‎

-7 Bytes
Binary file not shown.

‎data/docs/bulk-submissions/NMDC-EDGE-Metagenomics-pipeline-bulk-submission.xlsx‎

-91 Bytes
Binary file not shown.

‎webapp/server/crons/bulkSubmissionMonitor.js‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ const config = require("../config");
1111
const { workflowlist, pipelinelist } = require("../config/workflow");
1212

1313
const isValidProjectName = (name) => {
14+
if(!name || name.trim() === '') {
15+
return false;
16+
}
1417
const regexp = new RegExp(/^[a-zA-Z0-9\-_.]{3,30}$/)
1518
return regexp.test(name.trim())
1619
}
1720
const isValidSRAInput = (name, accession) => {
18-
return common.fileExistsSync(`${config.IO.SRA_BASE_DIR}/${accession}/${name}`);
21+
return fs.existsSync(`${config.IO.SRA_BASE_DIR}/${accession}/${name}`);
1922
}
2023

2124
const bulkSubmissionMonitor = async () => {
@@ -46,7 +49,10 @@ const bulkSubmissionMonitor = async () => {
4649
const bulkExcel = bulkSubmission_home + "/" + conf.bulkfile.name;
4750
// Parse a file
4851
const workSheetsFromFile = xlsx.parse(bulkExcel);
49-
let rows = workSheetsFromFile[0].data;
52+
let rows = workSheetsFromFile[0].data.filter(row => {
53+
// Check if all cells in the row are empty (null, undefined, or empty string after trim)
54+
return row.some(cell => cell !== null && cell !== undefined && String(cell).trim() !== '');
55+
});
5056
// Remove header
5157
rows.shift();
5258
// validate inputs
@@ -60,6 +66,11 @@ const bulkSubmissionMonitor = async () => {
6066
for (cols of rows) {
6167
let submission = {};
6268
currRow++;
69+
if(cols.length < 6) {
70+
validInput = false;
71+
errMsg += `ERROR: Row ${currRow}: Invalid input.\n`;
72+
continue;
73+
}
6374
// validate project name
6475
if (!isValidProjectName(cols[0])) {
6576
validInput = false;
@@ -69,16 +80,16 @@ const bulkSubmissionMonitor = async () => {
6980
submission['proj_desc'] = cols[1];
7081
}
7182
//get Sequencing Platform, default is Illumina
72-
if (!cols[5] || cols[5] === 'Illumina') {
73-
submission['platform'] = 'Illumina';
74-
submission['shortRead'] = true;
75-
} else {
83+
if (cols[5] && cols[5] === 'PacBio') {
7684
submission['platform'] = 'PacBio';
7785
submission['shortRead'] = false;
86+
} else {
87+
submission['platform'] = 'Illumina';
88+
submission['shortRead'] = true;
7889
}
7990

8091
if (cols[2] && cols[2].trim()) {
81-
// validate Paired-end Illumina/PacBio fastq and ignore the Illumina Pair-1/paire-2
92+
// validate Interleaved or Single-end Illumina/PacBio fastq and ignore the Illumina Pair-1/paire-2
8293
submission['interleaved'] = true;
8394
let fastqs = [];
8495
let fastqs_display = [];
@@ -94,7 +105,7 @@ const bulkSubmissionMonitor = async () => {
94105
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
95106
if (!file) {
96107
validInput = false;
97-
errMsg += `ERROR: Row ${currRow}: Paired-end Illumina/PacBio FASTQ ${fq} not found.\n`;
108+
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
98109
} else {
99110
fastqs.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
100111
fastqs_display.push(`uploads/${file.owner}/${fq}`);
@@ -108,11 +119,11 @@ const bulkSubmissionMonitor = async () => {
108119
fastqs_display.push(`sradata/${accession}/${fq}`);
109120
} else {
110121
validInput = false;
111-
errMsg += `ERROR: Row ${currRow}: Paired-end Illumina/PacBio FASTQ ${fq} not found.\n`;
122+
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
112123
}
113124
} else {
114125
validInput = false;
115-
errMsg += `ERROR: Row ${currRow}: Paired-end Illumina/PacBio FASTQ ${fq} not valid.\n`;
126+
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not valid.\n`;
116127
}
117128
}
118129
submission['input_fastqs'] = fastqs;
@@ -132,7 +143,7 @@ const bulkSubmissionMonitor = async () => {
132143
// validate the Illumina Pair-1/paire-2
133144
if (!(cols[3] && cols[3].trim())) {
134145
validInput = false;
135-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-1 FASTQ required.\n`;
146+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 required.\n`;
136147
} else {
137148
fq1s = cols[3].trim().split(/,/);
138149
for (fq of fq1s) {
@@ -146,7 +157,7 @@ const bulkSubmissionMonitor = async () => {
146157
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
147158
if (!file) {
148159
validInput = false;
149-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-1 FASTQ ${fq} not found.\n`;
160+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 ${fq} not found.\n`;
150161
} else {
151162
pairFq1.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
152163
pairFq1_display.push(`uploads/${file.owner}/${fq}`);
@@ -160,32 +171,32 @@ const bulkSubmissionMonitor = async () => {
160171
pairFq1_display.push(`sradata/${accession}/${fq}`);
161172
} else {
162173
validInput = false;
163-
errMsg += `ERROR: Row ${currRow}: Paired-end Illumina/PacBio FASTQ ${fq} not found.\n`;
174+
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
164175
}
165176
} else {
166177
validInput = false;
167-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-1 FASTQ ${fq} not valid.\n`;
178+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 ${fq} not valid.\n`;
168179
}
169180
};
170181
}
171182

172183
if (!(cols[4] && cols[4].trim())) {
173184
validInput = false;
174-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-2 FASTQ required.\n`;
185+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 required.\n`;
175186
} else {
176187
fq2s = cols[4].trim().split(/,/);
177188
for (fq of fq2s) {
178189
fq = fq.trim();
179190
if (fq.toUpperCase().startsWith('HTTP')) {
180191
pairFq2.push(fq);
181-
pairFq1_display.push(fq);
192+
pairFq2_display.push(fq);
182193
} else if (fq.toUpperCase().startsWith('UPLOAD')) {
183194
fq = fq.replace(uploadReg, '');
184195
// it's uploaded file
185196
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
186197
if (!file) {
187198
validInput = false;
188-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-2 FASTQ ${fq} not found.\n`;
199+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 ${fq} not found.\n`;
189200
} else {
190201
pairFq2.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
191202
pairFq2_display.push(`uploads/${file.owner}/${fq}`);
@@ -199,18 +210,18 @@ const bulkSubmissionMonitor = async () => {
199210
pairFq2_display.push(`sradata/${accession}/${fq}`);
200211
} else {
201212
validInput = false;
202-
errMsg += `ERROR: Row ${currRow}: Paired-end Illumina/PacBio FASTQ ${fq} not found.\n`;
213+
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
203214
}
204215
} else {
205216
validInput = false;
206-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-2 FASTQ ${fq} not valid.\n`;
217+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 ${fq} not valid.\n`;
207218
}
208219
};
209220
}
210221

211222
if (fq1s && fq2s && fq1s.length !== fq2s.length) {
212223
validInput = false;
213-
errMsg += `ERROR: Row ${currRow}: Illumina Pair-1 FASTQ and Illumina Pair-2 FASTQ have different input fastq file counts.\n`;
224+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 and Illumina Paired-end R2 have different input fastq file counts.\n`;
214225
}
215226
if (validInput) {
216227
submission['input_fastqs'] = [];

0 commit comments

Comments
 (0)