Displayed executor errors

Author: zonia3000

CHANGELOG.md

@@ -7,6 +7,7 @@
 * Updated slim-select version to support range selection (\#827);
 * Supported pre/post `pinned_package_versions` in task-collection (\#827);
 * Adapted frontend to new task-group deletion endpoint (\#833);
+* Displayed executor errors (\#833);
 
 # 1.19.3
 

__tests__/job_utilities.test.js

@@ -3,7 +3,8 @@ import {
 	extractJobErrorParts,
 	extractRelevantJobError,
 	generateNewUniqueDatasetName,
-	getFirstTaskIndexForContinuingWorkflow
+	getFirstTaskIndexForContinuingWorkflow,
+	showExecutorErrorLog
 } from '$lib/common/job_utilities.js';
 
 const completeTracebackError = `TASK ERROR:Task id: 15 (Create OME-Zarr structure), e.workflow_task_order=0
@@ -323,6 +324,27 @@ describe('get first task index for continuing workflow', () => {
 	});
 });
 
+it('should handle executor_error_log', () => {
+	const job = /** @type {import('fractal-components/types/api').ApplyWorkflowV2} */ ({});
+	expect(showExecutorErrorLog({ ...job, status: 'done', executor_error_log: null })).toBeFalsy();
+	expect(showExecutorErrorLog({ ...job, status: 'failed', executor_error_log: null })).toBeFalsy();
+	expect(
+		showExecutorErrorLog({
+			...job,
+			status: 'failed',
+			executor_error_log: `slurmstepd: error: Detected 1 oom_kill event in StepId=111.0. Some of the step tasks have been OOM Killed.
+		srun: error: u20-cva0000-113: task 0: Out Of Memory`
+		})
+	).toBeTruthy();
+	expect(
+		showExecutorErrorLog({
+			...job,
+			status: 'failed',
+			executor_error_log: `foobar`
+		})
+	).toBeFalsy();
+});
+
 /**
  * @param {string[]} names
  */

__tests__/v2/JobLogsModal.test.js

@@ -35,7 +35,7 @@ Command "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/bin/pytho
 		await result.component.show(mockApplyWorkflow({ id: 1, status: 'failed', log: null }), false);
 		const pre = /** @type {HTMLElement} */ (result.container.querySelector('pre'));
 		expect(pre.classList.contains('highlight')).eq(true);
-		expect(pre.innerHTML).eq(error);
+		expect(pre.querySelector('div')?.innerHTML).eq(error);
 	});
 
 	it('display log with highlighting and hidden details', async () => {
@@ -85,7 +85,7 @@ allowed_channels
 		await result.component.show(mockApplyWorkflow({ id: 1, status: 'done', log: null }), false);
 		const pre = /** @type {HTMLElement} */ (result.container.querySelector('pre'));
 		expect(pre.classList.contains('highlight')).eq(false);
-		expect(pre.innerHTML).eq(log);
+		expect(pre.querySelector('div')?.innerHTML).eq(log);
 	});
 
 	it('error while loading job for user', async () => {

components/src/lib/types/api.d.ts

@@ -146,6 +146,7 @@ export type ApplyWorkflowV2 = {
 	user_email: string;
 	slurm_account: string | null;
 	attribute_filters: { [key: string]: string | number | boolean };
+	executor_error_log: string | null;
 };
 
 export type JobStatus = 'submitted' | 'done' | 'failed';

playwright.config.js

@@ -88,7 +88,7 @@ export default defineConfig({
 
 	webServer: [
 		{
-			command: './tests/start-test-server.sh --branch 2743-delete-task-group-path-on-disk-during-task-group-deletion',
+			command: './tests/start-test-server.sh --branch main',
 			port: 8000,
 			waitForPort: true,
 			stdout: 'pipe',

src/lib/common/job_utilities.js

@@ -185,3 +185,20 @@ export function getFirstTaskIndexForContinuingWorkflow(
 
 	return undefined;
 }
+
+/**
+ * @param {import('fractal-components/types/api').ApplyWorkflowV2} job
+ */
+export function showExecutorErrorLog(job) {
+	if (job.status !== 'failed' || !job.executor_error_log) {
+		return false;
+	}
+	const log = job.executor_error_log.toLowerCase();
+	const errorKeywords = ['oom', 'killed', 'srun: error', 'out of memory', 'due to time limit'];
+	for (const keyword of errorKeywords) {
+		if (log.includes(keyword)) {
+			return true;
+		}
+	}
+	return false;
+}

src/lib/components/common/ExpandableLog.svelte

@@ -49,7 +49,7 @@
 						onclick={expandDetails}>... (details hidden, click here to expand)</button
 					>{/if}{/each}</pre>
 	{:else}
-		<pre class:highlight>{logParts.map((p) => p.text).join('\n')}</pre>
+		<pre class:highlight><div class="ps-3 pe-3">{logParts.map((p) => p.text).join('\n')}</div></pre>
 	{/if}
 </div>
 

src/lib/components/v2/jobs/JobLogsModal.svelte

@@ -1,6 +1,6 @@
 <script>
 	import { env } from '$env/dynamic/public';
-	import { extractJobErrorParts } from '$lib/common/job_utilities';
+	import { extractJobErrorParts, showExecutorErrorLog } from '$lib/common/job_utilities';
 	import { onDestroy } from 'svelte';
 	import Modal from '../../common/Modal.svelte';
 	import ExpandableLog from '$lib/components/common/ExpandableLog.svelte';
@@ -15,6 +15,8 @@
 	let admin = false;
 	let log = '';
 	let loading = $state(true);
+	/** @type {'main'|'slurm'} */
+	let selectedTab = $state('main');
 
 	const updateJobInterval = env.PUBLIC_UPDATE_JOBS_INTERVAL
 		? parseInt(env.PUBLIC_UPDATE_JOBS_INTERVAL)
@@ -31,6 +33,7 @@
 		if (errorAlert) {
 			errorAlert.hide();
 		}
+		selectedTab = showExecutorErrorLog(selectedJob) ? 'slurm' : 'main';
 		job = selectedJob;
 		admin = isAdminPage;
 		log = '';
@@ -124,14 +127,42 @@
 		</div>
 	{/snippet}
 	{#snippet body()}
-		<div id="errorAlert-workflowJobLogsModal"></div>
-		{#if loading}
-			<div class="spinner-border spinner-border-sm" role="status">
-				<span class="visually-hidden">Loading...</span>
-			</div>
-			Loading...
+		{#if job && showExecutorErrorLog(job)}
+			<ul class="nav nav-tabs mb-3">
+				<li class="nav-item">
+					<button
+						class="nav-link"
+						class:active={selectedTab === 'main'}
+						onclick={() => (selectedTab = 'main')}
+						aria-current={selectedTab === 'main' ? 'page' : undefined}
+					>
+						Main Log
+					</button>
+				</li>
+				<li class="nav-item">
+					<button
+						class="nav-link"
+						class:active={selectedTab === 'slurm'}
+						onclick={() => (selectedTab = 'slurm')}
+						aria-current={selectedTab === 'slurm' ? 'page' : undefined}
+					>
+						SLURM Error Log
+					</button>
+				</li>
+			</ul>
+		{/if}
+		{#if selectedTab === 'main'}
+			<div id="errorAlert-workflowJobLogsModal"></div>
+			{#if loading}
+				<div class="spinner-border spinner-border-sm" role="status">
+					<span class="visually-hidden">Loading...</span>
+				</div>
+				Loading...
+			{:else}
+				<ExpandableLog bind:logParts highlight={job?.status === 'failed'} />
+			{/if}
 		{:else}
-			<ExpandableLog bind:logParts highlight={job?.status === 'failed'} />
+			<pre><div class="ps-3 pe-3">{job?.executor_error_log}</div></pre>
 		{/if}
 	{/snippet}
 </Modal>

src/routes/v2/projects/[projectId]/workflows/[workflowId]/+page.svelte

@@ -12,7 +12,7 @@
 	import VersionUpdate from '$lib/components/v2/workflow/VersionUpdate.svelte';
 	import ImagesStatus from '$lib/components/jobs/ImagesStatus.svelte';
 	import TasksOrderModal from '$lib/components/v2/workflow/TasksOrderModal.svelte';
-	import { extractRelevantJobError } from '$lib/common/job_utilities';
+	import { extractRelevantJobError, showExecutorErrorLog } from '$lib/common/job_utilities';
 	import JobLogsModal from '$lib/components/v2/jobs/JobLogsModal.svelte';
 	import TaskInfoTab from '$lib/components/v2/workflow/TaskInfoTab.svelte';
 	import InputFiltersTab from '$lib/components/v2/workflow/InputFiltersTab.svelte';
@@ -49,7 +49,7 @@
 
 	let jobError = $state('');
 	/** @type {import('fractal-components/types/api').ApplyWorkflowV2|undefined} */
-	let failedJob;
+	let failedJob = $state();
 	/** @type {JobLogsModal|undefined} */
 	let jobLogsModal = $state();
 
@@ -821,7 +821,11 @@
 						<div class="text-muted mb-2 fw-bolder">
 							The last job failed with the following error:
 						</div>
-						<pre class="text-danger mb-0">{jobError}</pre>
+						{#if failedJob && showExecutorErrorLog(failedJob)}
+							<pre class="text-danger mb-0">{failedJob.executor_error_log}</pre>
+						{:else}
+							<pre class="text-danger mb-0">{jobError}</pre>
+						{/if}
 					</div>
 					<div class="col-md-2 col-sm-3">
 						<button class="btn btn-outline-secondary float-end" onclick={showJobLogsModal}>