Skip to content

Commit 0dbe35a

Browse files
mflynn-lanlmflynn-lanl
authored
Merge pull request #414 from microbiomedata/412-update-bulk-submission-templates
Add "Data Source" column to template
2 parents 270af4f + a96b84e commit 0dbe35a

File tree

3 files changed

+44
-60
lines changed

3 files changed

+44
-60
lines changed

‎data/docs/bulk-submissions/NMDC-EDGE-Metagenomics-ReadsQC-bulk-submission.xlsx‎

66 Bytes
Binary file not shown.

‎data/docs/bulk-submissions/NMDC-EDGE-Metagenomics-pipeline-bulk-submission.xlsx‎

66 Bytes
Binary file not shown.

‎webapp/server/crons/bulkSubmissionMonitor.js‎

Lines changed: 44 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,10 @@ const bulkSubmissionMonitor = async () => {
6060
let errMsg = '';
6161
let currRow = 1;
6262
let submissions = [];
63-
const uploadReg = /^upload\//i;
64-
const sraReg = /^sra\//i;
6563

6664
for (cols of rows) {
6765
let submission = {};
66+
let dataSource = cols[2] ? cols[2].trim() : 'Uploaded File';
6867
currRow++;
6968
if(cols.length < 6) {
7069
validInput = false;
@@ -80,38 +79,26 @@ const bulkSubmissionMonitor = async () => {
8079
submission['proj_desc'] = cols[1];
8180
}
8281
//get Sequencing Platform, default is Illumina
83-
if (cols[5] && cols[5] === 'PacBio') {
82+
if (cols[6] && cols[6] === 'PacBio') {
8483
submission['platform'] = 'PacBio';
8584
submission['shortRead'] = false;
8685
} else {
8786
submission['platform'] = 'Illumina';
8887
submission['shortRead'] = true;
8988
}
9089

91-
if (cols[2] && cols[2].trim()) {
90+
if (cols[3] && cols[3].trim()) {
9291
// validate Interleaved or Single-end Illumina/PacBio fastq and ignore the Illumina Pair-1/paire-2
9392
submission['interleaved'] = true;
9493
let fastqs = [];
9594
let fastqs_display = [];
96-
const fqs = cols[2].trim().split(/,/);
95+
const fqs = cols[3].split(/,/);
9796
for (fq of fqs) {
9897
fq = fq.trim();
99-
if (fq.toUpperCase().startsWith('HTTP')) {
98+
if (dataSource === 'HTTP(s) URL') {
10099
fastqs.push(fq);
101100
fastqs_display.push(fq);
102-
} else if (fq.toUpperCase().startsWith('UPLOAD')) {
103-
fq = fq.replace(uploadReg, '');
104-
// it's uploaded file
105-
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
106-
if (!file) {
107-
validInput = false;
108-
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
109-
} else {
110-
fastqs.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
111-
fastqs_display.push(`uploads/${file.owner}/${fq}`);
112-
}
113-
} else if (fq.toUpperCase().startsWith('SRA')) {
114-
fq = fq.replace(sraReg, '');
101+
} else if (dataSource === 'Retrieved SRA Data') {
115102
const regex = /[\._]/; // Split on dot and _
116103
const accession = fq.split(regex)[0];
117104
if(isValidSRAInput(fq, accession)) {
@@ -122,9 +109,16 @@ const bulkSubmissionMonitor = async () => {
122109
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
123110
}
124111
} else {
125-
validInput = false;
126-
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not valid.\n`;
127-
}
112+
// it's uploaded file
113+
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
114+
if (!file) {
115+
validInput = false;
116+
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
117+
} else {
118+
fastqs.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
119+
fastqs_display.push(`uploads/${file.owner}/${fq}`);
120+
}
121+
}
128122
}
129123
submission['input_fastqs'] = fastqs;
130124
submission['input_fastqs_display'] = fastqs_display;
@@ -141,29 +135,17 @@ const bulkSubmissionMonitor = async () => {
141135
let fq1s = null;
142136
let fq2s = null;
143137
// validate the Illumina Pair-1/paire-2
144-
if (!(cols[3] && cols[3].trim())) {
138+
if (!(cols[4] && cols[4].trim())) {
145139
validInput = false;
146140
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 required.\n`;
147141
} else {
148-
fq1s = cols[3].trim().split(/,/);
142+
fq1s = cols[4].split(/,/);
149143
for (fq of fq1s) {
150144
fq = fq.trim();
151-
if (fq.toUpperCase().startsWith('HTTP')) {
145+
if (dataSource === 'HTTP(s) URL') {
152146
pairFq1.push(fq);
153147
pairFq1_display.push(fq);
154-
} else if (fq.toUpperCase().startsWith('UPLOAD')) {
155-
fq = fq.replace(uploadReg, '');
156-
// it's uploaded file
157-
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
158-
if (!file) {
159-
validInput = false;
160-
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 ${fq} not found.\n`;
161-
} else {
162-
pairFq1.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
163-
pairFq1_display.push(`uploads/${file.owner}/${fq}`);
164-
}
165-
} else if (fq.toUpperCase().startsWith('SRA')) {
166-
fq = fq.replace(sraReg, '');
148+
} else if (dataSource === 'Retrieved SRA Data') {
167149
const regex = /[\._]/; // Split on dot and _
168150
const accession = fq.split(regex)[0];
169151
if(isValidSRAInput(fq, accession)) {
@@ -174,35 +156,30 @@ const bulkSubmissionMonitor = async () => {
174156
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
175157
}
176158
} else {
177-
validInput = false;
178-
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 ${fq} not valid.\n`;
179-
}
159+
// it's uploaded file
160+
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
161+
if (!file) {
162+
validInput = false;
163+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R1 ${fq} not found.\n`;
164+
} else {
165+
pairFq1.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
166+
pairFq1_display.push(`uploads/${file.owner}/${fq}`);
167+
}
168+
}
180169
};
181170
}
182171

183-
if (!(cols[4] && cols[4].trim())) {
172+
if (!(cols[5] && cols[5].trim())) {
184173
validInput = false;
185174
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 required.\n`;
186175
} else {
187-
fq2s = cols[4].trim().split(/,/);
176+
fq2s = cols[5].split(/,/);
188177
for (fq of fq2s) {
189178
fq = fq.trim();
190-
if (fq.toUpperCase().startsWith('HTTP')) {
179+
if (dataSource === 'HTTP(s) URL') {
191180
pairFq2.push(fq);
192181
pairFq2_display.push(fq);
193-
} else if (fq.toUpperCase().startsWith('UPLOAD')) {
194-
fq = fq.replace(uploadReg, '');
195-
// it's uploaded file
196-
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
197-
if (!file) {
198-
validInput = false;
199-
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 ${fq} not found.\n`;
200-
} else {
201-
pairFq2.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
202-
pairFq2_display.push(`uploads/${file.owner}/${fq}`);
203-
}
204-
} else if (fq.toUpperCase().startsWith('SRA')) {
205-
fq = fq.replace(sraReg, '');
182+
} else if (dataSource === 'Retrieved SRA Data') {
206183
const regex = /[\._]/; // Split on dot and _
207184
const accession = fq.split(regex)[0];
208185
if(isValidSRAInput(fq, accession)) {
@@ -213,9 +190,16 @@ const bulkSubmissionMonitor = async () => {
213190
errMsg += `ERROR: Row ${currRow}: Interleaved or Single-end Illumina/PacBio FASTQ ${fq} not found.\n`;
214191
}
215192
} else {
216-
validInput = false;
217-
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 ${fq} not valid.\n`;
218-
}
193+
// it's uploaded file
194+
const file = await Upload.findOne({ name: { $eq: fq }, status: { $ne: 'delete' } });
195+
if (!file) {
196+
validInput = false;
197+
errMsg += `ERROR: Row ${currRow}: Illumina Paired-end R2 ${fq} not found.\n`;
198+
} else {
199+
pairFq2.push(`${config.IO.UPLOADED_FILES_DIR}/${file.code}`);
200+
pairFq2_display.push(`uploads/${file.owner}/${fq}`);
201+
}
202+
}
219203
};
220204
}
221205

0 commit comments

Comments
 (0)