Skip to content

Commit 0a274d6

Browse files
abhi18avabhi18av
authored
Merge pull request #58 from nextflow-io/add-logs
log detail errors from remote
2 parents 09a9e68 + 488f3ce commit 0a274d6

File tree

3 files changed

+89
-21
lines changed

3 files changed

+89
-21
lines changed

‎plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy‎

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ class NomadService implements Closeable{
109109
JobRegisterResponse jobRegisterResponse = jobsApi.registerJob(jobRegisterRequest, config.jobOpts().region, config.jobOpts().namespace, null, null)
110110
jobRegisterResponse.evalID
111111
} catch (Throwable e) {
112+
log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e)
112113
throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e)
113114
}
114115

@@ -267,27 +268,42 @@ class NomadService implements Closeable{
267268
}
268269

269270
String getJobState(String jobId){
270-
List<AllocationListStub> allocations = jobsApi.getJobAllocations(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null, null)
271-
AllocationListStub last = allocations?.sort{
272-
it.modifyIndex
273-
}?.last()
274-
String currentState = last?.taskStates?.values()?.last()?.state
275-
log.debug "Task $jobId , state=$currentState"
276-
currentState ?: "Unknown"
271+
try {
272+
List<AllocationListStub> allocations = jobsApi.getJobAllocations(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null, null)
273+
AllocationListStub last = allocations?.sort {
274+
it.modifyIndex
275+
}?.last()
276+
String currentState = last?.taskStates?.values()?.last()?.state
277+
log.debug "Task $jobId , state=$currentState"
278+
currentState ?: "Unknown"
279+
}catch(Exception e){
280+
log.debug("[NOMAD] Failed to get jobState ${jobId} -- Cause: ${e.message ?: e}", e)
281+
"dead"
282+
}
277283
}
278284

279285

280286

281287
boolean checkIfRunning(String jobId){
282-
Job job = jobsApi.getJob(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null)
283-
log.debug "[NOMAD] checkIfRunning jobID=$job.ID; status=$job.status"
284-
job.status == "running"
288+
try {
289+
Job job = jobsApi.getJob(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null)
290+
log.debug "[NOMAD] checkIfRunning jobID=$job.ID; status=$job.status"
291+
job.status == "running"
292+
}catch (Exception e){
293+
log.debug("[NOMAD] Failed to get jobState ${jobId} -- Cause: ${e.message ?: e}", e)
294+
false
295+
}
285296
}
286297

287298
boolean checkIfDead(String jobId){
288-
Job job = jobsApi.getJob(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null)
289-
log.debug "[NOMAD] checkIfDead jobID=$job.ID; status=$job.status"
290-
job.status == "dead"
299+
try{
300+
Job job = jobsApi.getJob(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null)
301+
log.debug "[NOMAD] checkIfDead jobID=$job.ID; status=$job.status"
302+
job.status == "dead"
303+
}catch (Exception e){
304+
log.debug("[NOMAD] Failed to get job ${jobId} -- Cause: ${e.message ?: e}", e)
305+
true
306+
}
291307
}
292308

293309
void kill(String jobId) {
@@ -300,12 +316,21 @@ class NomadService implements Closeable{
300316

301317
protected void purgeJob(String jobId, boolean purge){
302318
log.debug "[NOMAD] purgeJob with jobId=${jobId}"
303-
jobsApi.deleteJob(jobId,config.jobOpts().region, config.jobOpts().namespace,null,null,purge, true)
319+
try {
320+
jobsApi.deleteJob(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, purge, true)
321+
}catch(Exception e){
322+
log.debug("[NOMAD] Failed to delete job ${jobId} -- Cause: ${e.message ?: e}", e)
323+
}
304324
}
305325

306326
String getClientOfJob(String jobId) {
307-
List<AllocationListStub> allocations = jobsApi.getJobAllocations(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null, null)
308-
AllocationListStub jobAllocation = allocations.first()
309-
return jobAllocation.nodeName
327+
try{
328+
List<AllocationListStub> allocations = jobsApi.getJobAllocations(jobId, config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null, null)
329+
AllocationListStub jobAllocation = allocations.first()
330+
return jobAllocation.nodeName
331+
}catch (Exception e){
332+
log.debug("[NOMAD] Failed to get job allocations ${jobId} -- Cause: ${e.message ?: e}", e)
333+
throw new ProcessSubmitException("[NOMAD] Failed to get alloactions ${jobId} -- Cause: ${e.message ?: e}", e)
334+
}
310335
}
311336
}

‎plugins/nf-nomad/src/test/nextflow/nomad/MockScriptRunner.groovy‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,6 @@ class MockExecutor extends NomadExecutor {
124124
@Override
125125
void signal() { }
126126

127-
protected TaskMonitor createTaskMonitor2() {
128-
new MockMonitor()
129-
}
130-
131127
}
132128

133129
class MockMonitor implements TaskMonitor {

‎plugins/nf-nomad/src/test/nextflow/nomad/NomadDSLSpec.groovy‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import org.pf4j.PluginDescriptorFinder
3131
import spock.lang.Shared
3232
import spock.lang.Timeout
3333
import test.Dsl2Spec
34+
import test.OutputCapture
3435

3536
import java.nio.file.Files
3637
import java.nio.file.Path
@@ -157,4 +158,50 @@ class NomadDSLSpec extends Dsl2Spec{
157158
// submitted
158159
// summary
159160
}
161+
162+
@org.junit.Rule
163+
OutputCapture capture = new OutputCapture()
164+
165+
def 'should catch a remote exception' () {
166+
given:
167+
mockWebServer.dispatcher = new Dispatcher() {
168+
@Override
169+
MockResponse dispatch(@NotNull RecordedRequest recordedRequest) throws InterruptedException {
170+
new MockResponse().setResponseCode(500).setBody("Dummy exception")
171+
}
172+
}
173+
174+
when:
175+
def SCRIPT = '''
176+
process sayHello{
177+
container 'ubuntu:22.0.4'
178+
input:
179+
val x
180+
output:
181+
stdout
182+
script:
183+
"""
184+
echo '$x world!\'
185+
"""
186+
}
187+
workflow {
188+
channel.of('hi!') | sayHello | view
189+
}
190+
'''
191+
and:
192+
def result = new MockScriptRunner([
193+
process:[executor:'nomad'],
194+
nomad:
195+
[
196+
client:
197+
[
198+
address : "http://${mockWebServer.hostName}:${mockWebServer.port}"
199+
]
200+
]
201+
]).setScript(SCRIPT).execute()
202+
203+
then:
204+
thrown(AbortRunException) //it fails because no real task is executed
205+
capture.toString().indexOf("io.nomadproject.client.ApiException: Server Error") != -1
206+
}
160207
}

0 commit comments

Comments
 (0)