Merge pull request #402 from fractal-analytics-platform/highlight-error-message

Highlighted relevant part of error message in workflow job log modal

Author: zonia3000

CHANGELOG.md

@@ -1,5 +1,12 @@
 *Note: Numbers like (\#123) point to closed Pull Requests on the fractal-web repository.*
 
+# Unreleased
+
+* Improved sorting of users in dropdown of the admin jobs page (\#402).
+* Fixed bug in retrieval of job log from the admin jobs page (\#402).
+* Highlighted relevant part of the error message in workflow job log modal (\#402).
+* Made the error message directly accessible from the new workflow page (\#402).
+
 # 0.9.0
 
 This release requires fractal-server 1.4.3.

__tests__/admin_jobs_page.test.js

@@ -0,0 +1,44 @@
+import { describe, it, expect, vi } from 'vitest';
+import { render } from '@testing-library/svelte';
+import { readable } from 'svelte/store';
+
+// Mocking the page store
+vi.mock('$app/stores', () => {
+	return {
+		page: readable({
+			data: {
+				users: [
+					{ id: 1, email: '[email protected]' },
+					{ id: 2, email: '[email protected]' },
+					{ id: 3, email: '[email protected]' },
+					{ id: 4, email: '[email protected]' }
+				],
+				userInfo: {
+					id: 2
+				}
+			}
+		})
+	};
+});
+
+// Mocking public variables
+vi.mock('$env/dynamic/public', () => {
+	return { env: {} };
+});
+
+// The component to be tested must be imported after the mock setup
+import page from '../src/routes/admin/jobs/+page.svelte';
+
+describe('Admin jobs page', () => {
+	it('Users in dropdown are correctly sorted', async () => {
+		const result = render(page);
+		const dropdown = result.getByLabelText('User');
+		const options = dropdown.querySelectorAll('option');
+		expect(options.length).eq(5);
+		expect(options[0].text).eq('All');
+		expect(options[1].text).eq('[email protected]');
+		expect(options[2].text).eq('[email protected]');
+		expect(options[3].text).eq('[email protected]');
+		expect(options[4].text).eq('[email protected]');
+	});
+});

__tests__/job_utilities.test.js

@@ -0,0 +1,215 @@
+import { it, expect } from 'vitest';
+import { extractJobErrorParts, extractRelevantJobError } from '$lib/common/job_utilities.js';
+
+const completeTracebackError = `TASK ERROR:Task id: 15 (Create OME-Zarr structure), e.workflow_task_order=0
+TRACEBACK:
+2024-01-29 16:52:02,328; INFO; START create_ome_zarr task
+Traceback (most recent call last):
+  File "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/lib/python3.10/site-packages/fractal_tasks_core/tasks/create_ome_zarr.py", line 470, in <module>
+    run_fractal_task(
+  File "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/lib/python3.10/site-packages/fractal_tasks_core/tasks/_utils.py", line 79, in run_fractal_task
+    metadata_update = task_function(**pars)
+  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function
+  File "pydantic/main.py", line 341, in pydantic.main.BaseModel.__init__
+pydantic.error_wrappers.ValidationError: 1 validation error for CreateOmeZarr
+allowed_channels
+  field required (type=value_error.missing)
+`;
+
+const shortTracebackError = `TASK ERROR:Task id: 20 (Create OME-Zarr structure), e.workflow_task_order=0
+TRACEBACK:
+Command "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/bin/python" is not valid. Hint: make sure that it is executable.`;
+
+it('detect parts of a workflow task error message with complete traceback', () => {
+	const parts = extractJobErrorParts(completeTracebackError);
+	expect(parts.length).eq(3);
+	expect(parts[0].text).eq(
+		'TASK ERROR:Task id: 15 (Create OME-Zarr structure), e.workflow_task_order=0'
+	);
+	expect(parts[0].highlight).eq(true);
+	expect(parts[1].text).eq(`TRACEBACK:
+2024-01-29 16:52:02,328; INFO; START create_ome_zarr task
+Traceback (most recent call last):
+  File "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/lib/python3.10/site-packages/fractal_tasks_core/tasks/create_ome_zarr.py", line 470, in <module>
+    run_fractal_task(
+  File "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/lib/python3.10/site-packages/fractal_tasks_core/tasks/_utils.py", line 79, in run_fractal_task
+    metadata_update = task_function(**pars)
+  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function
+  File "pydantic/main.py", line 341, in pydantic.main.BaseModel.__init__`);
+	expect(parts[1].highlight).eq(false);
+	expect(parts[2].text)
+		.eq(`pydantic.error_wrappers.ValidationError: 1 validation error for CreateOmeZarr
+allowed_channels
+  field required (type=value_error.missing)`);
+	expect(parts[2].highlight).eq(true);
+});
+
+it('detect parts of a workflow task error message with short traceback', () => {
+	const parts = extractJobErrorParts(shortTracebackError);
+	expect(parts.length).eq(3);
+	expect(parts[0].text).eq(
+		'TASK ERROR:Task id: 20 (Create OME-Zarr structure), e.workflow_task_order=0'
+	);
+	expect(parts[0].highlight).eq(true);
+	expect(parts[1].text).eq('TRACEBACK:');
+	expect(parts[1].highlight).eq(false);
+	expect(parts[2].text).eq(
+		`Command "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/bin/python" is not valid. Hint: make sure that it is executable.`
+	);
+	expect(parts[2].highlight).eq(true);
+});
+
+it('detect parts of a workflow task error message without traceback', () => {
+	const parts = extractJobErrorParts('foo');
+	expect(parts.length).eq(1);
+	expect(parts[0].text).eq('foo');
+	expect(parts[0].highlight).eq(false);
+});
+
+it('extract relevant part of a workflow task error message with complete traceback', () => {
+	const relevantError = extractRelevantJobError(completeTracebackError);
+	expect(relevantError)
+		.eq(`TASK ERROR:Task id: 15 (Create OME-Zarr structure), e.workflow_task_order=0
+pydantic.error_wrappers.ValidationError: 1 validation error for CreateOmeZarr
+allowed_channels
+  field required (type=value_error.missing)`);
+});
+
+it('extract relevant part of a workflow task error message with short traceback', () => {
+	const relevantError = extractRelevantJobError(shortTracebackError);
+	expect(relevantError).eq(
+		`TASK ERROR:Task id: 20 (Create OME-Zarr structure), e.workflow_task_order=0
+Command "/tmp/FRACTAL_TASKS_DIR/.fractal/fractal-tasks-core0.14.1/venv/bin/python" is not valid. Hint: make sure that it is executable.`
+	);
+});
+
+it('extract relevant part of a workflow task error message without traceback', () => {
+	const relevantError = extractRelevantJobError('foo');
+	expect(relevantError).eq('foo');
+});
+
+it('extract relevant part of a workflow task error with max lines reached', () => {
+	const relevantError = extractRelevantJobError(completeTracebackError, 3);
+	expect(relevantError)
+		.eq(`TASK ERROR:Task id: 15 (Create OME-Zarr structure), e.workflow_task_order=0
+pydantic.error_wrappers.ValidationError: 1 validation error for CreateOmeZarr
+allowed_channels
+[...]`);
+});
+
+it('extract relevant part of a workflow task error with max lines not reached', () => {
+	const relevantError = extractRelevantJobError(completeTracebackError, 4);
+	expect(relevantError)
+		.eq(`TASK ERROR:Task id: 15 (Create OME-Zarr structure), e.workflow_task_order=0
+pydantic.error_wrappers.ValidationError: 1 validation error for CreateOmeZarr
+allowed_channels
+  field required (type=value_error.missing)`);
+});
+
+it('detect parts of a workflow JobExecutionError', () => {
+	const error = `JOB ERROR:
+TRACEBACK:
+JobExecutionError
+
+Job cancelled due to executor shutdown.`;
+	const parts = extractJobErrorParts(error);
+	expect(parts.length).eq(3);
+	expect(parts[0].text).eq('JOB ERROR:');
+	expect(parts[0].highlight).eq(true);
+	expect(parts[1].text).eq('TRACEBACK:');
+	expect(parts[1].highlight).eq(false);
+	expect(parts[2].text).eq(`JobExecutionError
+
+Job cancelled due to executor shutdown.`);
+	expect(parts[2].highlight).eq(true);
+});
+
+it('detect parts of an error with multiple tracebacks', () => {
+	const log = `TASK ERROR:Task id: 1400 (Measure Features), e.workflow_task_order=10
+TRACEBACK:
+Traceback (most recent call last):
+  File "/redacted/anaconda3/envs/fractal-server-1.4.0/lib/python3.10/site-packages/fractal_server/app/runner/_common.py", line 408, in call_single_parallel_task
+    raise e
+  File "/redacted/anaconda3/envs/fractal-server-1.4.0/lib/python3.10/site-packages/fractal_server/app/runner/_common.py", line 401, in call_single_parallel_task
+    _call_command_wrapper(
+  File "/redacted/anaconda3/envs/fractal-server-1.4.0/lib/python3.10/site-packages/fractal_server/app/runner/_common.py", line 184, in _call_command_wrapper
+    raise TaskExecutionError(err)
+fractal_server.app.runner.common.TaskExecutionError: 2024-02-05 15:56:19,240; INFO; START measure_features task
+Traceback (most recent call last):
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/zarr/core.py", line 252, in _load_metadata_nosync
+    meta_bytes = self._store[mkey]
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/zarr/storage.py", line 1113, in __getitem__
+    raise KeyError(key)
+KeyError: '.zarray'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/apx_fractal_task_collection/measure_features.py", line 299, in <module>
+    run_fractal_task(
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/fractal_tasks_core/tasks/_utils.py", line 79, in run_fractal_task
+    metadata_update = task_function(**pars)
+  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function
+  File "pydantic/decorator.py", line 134, in pydantic.decorator.ValidatedFunction.call
+  File "pydantic/decorator.py", line 206, in pydantic.decorator.ValidatedFunction.execute
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/apx_fractal_task_collection/measure_features.py", line 211, in measure_features
+    label_image = da.from_zarr(
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/dask/array/core.py", line 3599, in from_zarr
+    z = zarr.Array(store, read_only=True, path=component, **kwargs)
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/zarr/core.py", line 224, in __init__
+    self._load_metadata()
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/zarr/core.py", line 243, in _load_metadata
+    self._load_metadata_nosync()
+  File "/redacted/fractal-demos/examples/server/FRACTAL_TASKS_DIR/.fractal/apx_fractal_task_collection0.1.1/venv/lib/python3.10/site-packages/zarr/core.py", line 254, in _load_metadata_nosync
+    raise ArrayNotFoundError(self._path)
+zarr.errors.ArrayNotFoundError: array not found at path %r' ''`;
+	const parts = extractJobErrorParts(log);
+	expect(parts.length).eq(3);
+	expect(parts[0].text).eq('TASK ERROR:Task id: 1400 (Measure Features), e.workflow_task_order=10');
+	expect(parts[0].highlight).eq(true);
+	expect(parts[1].highlight).eq(false);
+	expect(parts[2].text).eq("zarr.errors.ArrayNotFoundError: array not found at path %r' ''");
+	expect(parts[2].highlight).eq(true);
+});
+
+it('detect parts of an unknown error without traceback', () => {
+	const log = `UNKNOWN ERROR
+Original error: something went really wrong`;
+	const parts = extractJobErrorParts(log);
+	expect(parts.length).eq(2);
+	expect(parts[0].text).eq('UNKNOWN ERROR');
+	expect(parts[0].highlight).eq(true);
+	expect(parts[1].text).eq('Original error: something went really wrong');
+	expect(parts[1].highlight).eq(false);
+});
+
+it('detect parts of an unknown error with traceback', () => {
+	const log = `UNKNOWN ERROR
+Original error: Traceback (most recent call last):
+  File "/redacted/anaconda3/envs/fractal-server-1.4.0/lib/python3.10/site-packages/fractal_server/app/runner/_common.py", line 281, in call_single_task
+    raise e
+  File "/redacted/anaconda3/envs/fractal-server-1.4.0/lib/python3.10/site-packages/fractal_server/app/runner/_common.py", line 274, in call_single_task
+    _call_command_wrapper(
+  File "/redacted/anaconda3/envs/fractal-server-1.4.0/lib/python3.10/site-packages/fractal_server/app/runner/_common.py", line 184, in _call_command_wrapper
+    raise TaskExecutionError(err)
+fractal_server.app.runner.common.TaskExecutionError: Traceback (most recent call last):
+  File "/redacted/apx_fractal_task_collection/src/apx_fractal_task_collection/multiplexed_pixel_clustering.py", line 26, in <module>
+    from sklearn.preprocessing import StandardScalerfrom, PowerTransformer
+ImportError: cannot import name 'StandardScalerfrom' from 'sklearn.preprocessing' (/redacted/.conda/envs/custom_fractal_task_collection/lib/python3.10/site-packages/sklearn/preprocessing/__init__.py)`;
+	const parts = extractJobErrorParts(log);
+	expect(parts.length).eq(3);
+	expect(parts[0].text).eq('UNKNOWN ERROR');
+	expect(parts[0].highlight).eq(true);
+	expect(parts[1].highlight).eq(false);
+	expect(parts[2].text).eq(
+		"ImportError: cannot import name 'StandardScalerfrom' from 'sklearn.preprocessing' (/redacted/.conda/envs/custom_fractal_task_collection/lib/python3.10/site-packages/sklearn/preprocessing/__init__.py)"
+	);
+	expect(parts[2].highlight).eq(true);
+});
+
+it('handles null or undefined inputs', () => {
+	expect(extractJobErrorParts(null)).toHaveLength(0);
+	expect(extractJobErrorParts()).toHaveLength(0);
+	expect(extractRelevantJobError(null)).eq('');
+	expect(extractRelevantJobError()).eq('');
+});

playwright.config.js

@@ -39,7 +39,7 @@ export default defineConfig({
 
 	webServer: [
 		{
-			command: './tests/start-test-server.sh 1.4.3a2',
+			command: './tests/start-test-server.sh 1.4.3',
 			port: 8000,
 			waitForPort: true,
 			stdout: 'pipe',

src/lib/common/job_utilities.js

@@ -0,0 +1,112 @@
+/**
+ * Split the error of a failed workflow job into multiple parts, marking the relevents ones,
+ * so that they can be extracted or highlighted in a different way in the UI.
+ *
+ * @param {string|null} log
+ * @returns {Array<{text: string, highlight: boolean}>}
+ */
+export function extractJobErrorParts(log) {
+	if (!log) {
+		return [];
+	}
+	log = log.trim();
+	if (
+		log.startsWith('TASK ERROR') ||
+		log.startsWith('JOB ERROR') ||
+		log.startsWith('UNKNOWN ERROR')
+	) {
+		const lines = log.split('\n');
+		if (lines.length > 1) {
+			const [firstLine, ...nextLines] = lines;
+			return [{ text: firstLine, highlight: true }, ...extractTraceback(nextLines.join('\n'))];
+		}
+	}
+	return [{ text: log, highlight: false }];
+}
+
+const completeTracebackLine = 'Traceback (most recent call last):';
+
+/**
+ * @param {string} error
+ */
+function extractTraceback(error) {
+	if (error.includes(completeTracebackLine)) {
+		return extractCompleteTraceback(error);
+	}
+	return extractUppercaseTraceback(error);
+}
+
+/**
+ * @param {string} error
+ * @returns {Array<{text: string, highlight: boolean}>}
+ */
+function extractCompleteTraceback(error) {
+	const index = error.lastIndexOf('Traceback (most recent call last):');
+	if (index !== -1) {
+		const firstPart = error.substring(0, index);
+		const lastPart = error.substring(index);
+		let relevantErrorStarted = false;
+		let part = firstPart;
+		const parts = [];
+		const lines = lastPart.split('\n');
+		for (let i = 0; i < lines.length; i++) {
+			const line = lines[i];
+			if (i > 0 && !relevantErrorStarted && !line.startsWith(' ') && !line.startsWith('\t')) {
+				parts.push({ text: part.trim(), highlight: false });
+				relevantErrorStarted = true;
+				part = '';
+			}
+			part += line + '\n';
+		}
+		parts.push({ text: part.trim(), highlight: true });
+		return parts;
+	}
+	return [{ text: error.trim(), highlight: false }];
+}
+
+/**
+ * @param {string} error
+ * @returns {Array<{text: string, highlight: boolean}>}
+ */
+function extractUppercaseTraceback(error) {
+	const uppercaseTraceback = 'TRACEBACK:';
+	const tracebackIndex = error.indexOf(uppercaseTraceback);
+	if (tracebackIndex !== -1) {
+		return [
+			{ text: error.substring(tracebackIndex, uppercaseTraceback.length), highlight: false },
+			{ text: error.substring(tracebackIndex + uppercaseTraceback.length + 1), highlight: true }
+		];
+	}
+	return [{ text: error.trim(), highlight: false }];
+}
+
+/**
+ * @param {string|null} completeJobError
+ * @param {number|undefined} maxLines
+ * @returns {string}
+ */
+export function extractRelevantJobError(completeJobError, maxLines = undefined) {
+	if (!completeJobError) {
+		return '';
+	}
+	const relevantParts = extractJobErrorParts(completeJobError).filter((p) => p.highlight);
+	let relevantError;
+	if (relevantParts.length === 0) {
+		relevantError = completeJobError;
+	} else {
+		relevantError = relevantParts.map((p) => p.text).join('\n');
+	}
+	if (!maxLines) {
+		return relevantError;
+	}
+	const lines = relevantError.split('\n');
+	if (lines.length > maxLines) {
+		const truncatedErrorLines = [];
+		for (let i = 0; i < maxLines; i++) {
+			truncatedErrorLines.push(lines[i]);
+		}
+		truncatedErrorLines.push('[...]');
+		return truncatedErrorLines.join('\n');
+	}
+	return relevantError;
+}