Merge pull request #467 from fractal-analytics-platform/import-export-dataset

Implemented import and export of datasets

Author: zonia3000

.github/workflows/end_to_end_tests.yaml

@@ -10,7 +10,7 @@ jobs:
   end_to_end_tests:
     name: 'Node ${{ matrix.node-version }}'
     runs-on: ubuntu-22.04
-    timeout-minutes: 10
+    timeout-minutes: 20
 
     strategy:
       matrix:

CHANGELOG.md

@@ -21,6 +21,8 @@
     * implemented continue/restart workflow (\#465);
     * set default first task when continuing a workflow (\#466);
     * displayed applied filters in workflow execution modal (\#466);
+    * implemented import and export of datasets (\#467);
+    * handled selection of default dataset on workflow page (\#467);
 
 # 0.10.2
 

__tests__/v2/workflow_utilities.test.js

@@ -0,0 +1,75 @@
+import { describe, it, expect } from 'vitest';
+import {
+	deleteDatasetSelectionsForProject,
+	getDefaultWorkflowDataset,
+	getSelectedWorkflowDataset,
+	saveSelectedDataset
+} from '$lib/common/workflow_utilities.js';
+
+describe('Workflow Utilities', () => {
+	it('Default dataset is taken from most recent job', () => {
+		const id = getDefaultWorkflowDataset(
+			[],
+			[
+				{ start_timestamp: '2024-04-24T10:18:37.868655+00:00', dataset_id: 1 },
+				{ start_timestamp: '2024-04-24T15:18:37.868655+00:00', dataset_id: 2 }
+			]
+		);
+		expect(id).toEqual(2);
+	});
+
+	it('Default dataset is taken from most recent dataset', () => {
+		const id = getDefaultWorkflowDataset(
+			[
+				{ id: 1, timestamp_created: '2024-04-24T10:18:37.868655+00:00' },
+				{ id: 2, timestamp_created: '2024-04-24T15:18:37.868655+00:00' }
+			],
+			[]
+		);
+		expect(id).toEqual(2);
+	});
+
+	it('Default dataset is undefined if there are no jobs and no datasets', () => {
+		const id = getDefaultWorkflowDataset([], []);
+		expect(id).toEqual(undefined);
+	});
+
+	it('Store and retrieve selected dataset from localStorage', () => {
+		const workflow1 = { id: 1, project_id: 1 };
+		const workflow2 = { id: 2, project_id: 1 };
+		const datasets = [{ id: 1 }, { id: 2 }, { id: 3 }];
+		let datasetId = getSelectedWorkflowDataset(workflow1, datasets, 3);
+		// return default dataset if nothing is stored in localStorage
+		expect(datasetId).toEqual(3);
+
+		saveSelectedDataset(workflow1, 2);
+		datasetId = getSelectedWorkflowDataset(workflow1, datasets, 3);
+		// return dataset saved in localStorage
+		expect(datasetId).toEqual(2);
+
+		saveSelectedDataset(workflow1, 4); // dataset doesn't exist
+		datasetId = getSelectedWorkflowDataset(workflow1, datasets, 3);
+		// return default dataset if the dataset stored in localstorage doesn't exist
+		expect(datasetId).toEqual(3);
+
+		saveSelectedDataset(workflow1, 1); // replace stored selection
+		datasetId = getSelectedWorkflowDataset(workflow1, datasets, 3);
+		// return dataset saved in localStorage
+		expect(datasetId).toEqual(1);
+
+		saveSelectedDataset(workflow2, 2); // add new selection
+		datasetId = getSelectedWorkflowDataset(workflow2, datasets, 3);
+		// return dataset saved in localStorage
+		expect(datasetId).toEqual(2);
+
+		saveSelectedDataset(workflow1, undefined); // remove stored value
+		datasetId = getSelectedWorkflowDataset(workflow1, datasets, 3);
+		// return default dataset
+		expect(datasetId).toEqual(3);
+
+		deleteDatasetSelectionsForProject(1); // remove all selections for the project
+		datasetId = getSelectedWorkflowDataset(workflow2, datasets, 3);
+		// return default dataset
+		expect(datasetId).toEqual(3);
+	});
+});

src/lib/common/workflow_utilities.js

@@ -0,0 +1,104 @@
+/**
+ * @param {Array<import("$lib/types-v2").DatasetV2>} datasets
+ * @param {Array<import("$lib/types-v2").ApplyWorkflowV2>} jobs
+ * @returns {number|undefined}
+ */
+export function getDefaultWorkflowDataset(datasets, jobs) {
+	if (jobs.length > 0) {
+		return jobs.sort((j1, j2) => (j1.start_timestamp > j2.start_timestamp ? -1 : 1))[0].dataset_id;
+	}
+	if (datasets.length > 0) {
+		return datasets.sort((d1, d2) => (d1.timestamp_created > d2.timestamp_created ? -1 : 1))[0].id;
+	}
+	return undefined;
+}
+
+/**
+ * @param {import("$lib/types-v2").WorkflowV2} workflow
+ * @param {Array<import("$lib/types-v2").DatasetV2>} datasets
+ * @param {number|undefined} defaultDatasetId
+ * @returns {number|undefined}
+ */
+export function getSelectedWorkflowDataset(workflow, datasets, defaultDatasetId) {
+	const selectedFromLocalStorage = getDatasetIdFromLocalStorage(workflow);
+	if (selectedFromLocalStorage === undefined) {
+		return defaultDatasetId;
+	}
+	const datasetExists = datasets.filter((d) => d.id === selectedFromLocalStorage).length > 0;
+	if (datasetExists) {
+		return selectedFromLocalStorage;
+	}
+	saveSelectedDataset(workflow, undefined);
+	return defaultDatasetId;
+}
+
+const LOCAL_STORAGE_SELECTED_DATASETS = 'SelectedDatasets';
+
+/**
+ * @param {import("$lib/types-v2").WorkflowV2} workflow
+ * @param {number|undefined} datasetId
+ */
+export function saveSelectedDataset(workflow, datasetId) {
+	let savedSelections = getDatasetSelectionsFromLocalStorage();
+	const selection = savedSelections.filter(
+		(s) => s.project_id === workflow.project_id && s.workflow_id === workflow.id
+	);
+	if (datasetId === undefined) {
+		savedSelections = savedSelections.filter((s) => s.workflow_id !== workflow.id);
+	} else if (selection.length > 0) {
+		savedSelections = savedSelections.map((s) =>
+			s.project_id === workflow.project_id && s.workflow_id === workflow.id
+				? { ...s, dataset_id: datasetId }
+				: s
+		);
+	} else {
+		savedSelections.push({
+			project_id: workflow.project_id,
+			workflow_id: workflow.id,
+			dataset_id: datasetId
+		});
+	}
+	localStorage.setItem(LOCAL_STORAGE_SELECTED_DATASETS, JSON.stringify(savedSelections));
+}
+
+/**
+ * @param {number} projectId
+ */
+export function deleteDatasetSelectionsForProject(projectId) {
+	let savedSelections = getDatasetSelectionsFromLocalStorage();
+	savedSelections = savedSelections.filter((s) => s.project_id !== projectId);
+	localStorage.setItem(LOCAL_STORAGE_SELECTED_DATASETS, JSON.stringify(savedSelections));
+}
+
+/**
+ * @param {import("$lib/types-v2").WorkflowV2} workflow
+ * @returns {number|undefined}
+ */
+function getDatasetIdFromLocalStorage(workflow) {
+	const savedSelections = getDatasetSelectionsFromLocalStorage();
+	const selection = savedSelections.filter(
+		(s) => s.project_id === workflow.project_id && s.workflow_id === workflow.id
+	);
+	if (selection.length > 0) {
+		return selection[0].dataset_id;
+	}
+	return undefined;
+}
+
+/**
+ * @returns {Array<{project_id: number, workflow_id: number, dataset_id: number}>}
+ */
+function getDatasetSelectionsFromLocalStorage() {
+	const storageContent = window.localStorage.getItem(LOCAL_STORAGE_SELECTED_DATASETS);
+	if (storageContent) {
+		try {
+			const parsedValue = JSON.parse(storageContent);
+			if (Array.isArray(parsedValue)) {
+				return parsedValue;
+			}
+		} catch (ex) {
+			console.error(`Invalid JSON inside localStorage item ${LOCAL_STORAGE_SELECTED_DATASETS}`);
+		}
+	}
+	return [];
+}

src/lib/components/common/jschema/schema_management.js

@@ -541,6 +541,7 @@ export function stripSchemaProperties(schema, legacy = true) {
 	if (schema.required) {
 		schema.required = schema.required.filter((k) => !ignoreProperties.includes(k));
 	}
+	return schema;
 }
 
 /**

src/lib/components/v2/projects/CreateWorkflowModal.svelte

@@ -4,7 +4,7 @@
 	import Modal from '../../common/Modal.svelte';
 	import { goto } from '$app/navigation';
 
-	/** @type {(workflow: import('$lib/types').Workflow) => void} */
+	/** @type {(workflow: import('$lib/types-v2').WorkflowV2) => void} */
 	export let handleWorkflowImported;
 
 	// Component properties

src/lib/components/v2/projects/ProjectDatasetsList.svelte

@@ -14,7 +14,9 @@
 	);
 
 	function createDatasetCallback(/** @type {import('$lib/types-v2').DatasetV2} */ newDataset) {
-		datasets = [...datasets, newDataset];
+		datasets = [...datasets, newDataset].sort((a, b) =>
+			a.name < b.name ? -1 : a.name > b.name ? 1 : 0
+		);
 	}
 
 	/**

src/lib/components/v2/projects/ProjectsList.svelte

@@ -4,6 +4,7 @@
 	import { AlertError, getValidationMessagesMap } from '$lib/common/errors';
 	import { goto } from '$app/navigation';
 	import Modal from '../../common/Modal.svelte';
+	import { deleteDatasetSelectionsForProject } from '$lib/common/workflow_utilities';
 
 	// List of projects to be displayed
 	/** @type {import('$lib/types').Project[]} */
@@ -92,6 +93,7 @@
 			console.log('Project deleted successfully');
 			// If the response is successful
 			projects = projects.filter((p) => p.id !== projectId);
+			deleteDatasetSelectionsForProject(projectId);
 		} else {
 			const result = await response.json();
 			console.error(`Unable to delete project ${projectId}`);

src/lib/components/v2/projects/WorkflowsList.svelte

@@ -4,9 +4,10 @@
 	import { AlertError } from '$lib/common/errors';
 	import CreateWorkflowModal from './CreateWorkflowModal.svelte';
 	import { onMount } from 'svelte';
+	import { saveSelectedDataset } from '$lib/common/workflow_utilities';
 
 	// The list of workflows
-	/** @type {import('$lib/types').Workflow[]} */
+	/** @type {import('$lib/types-v2').WorkflowV2[]} */
 	export let workflows = [];
 	// Set the projectId prop to reference a specific project for each workflow
 	export let projectId = undefined;
@@ -33,9 +34,9 @@
 
 		if (response.ok) {
 			console.log('Workflow deleted');
-			workflows = workflows.filter((wkf) => {
-				return wkf.id !== workflowId;
-			});
+			const deletedWorkflow = workflows.filter((w) => w.id === workflowId)[0];
+			workflows = workflows.filter((w) => w.id !== workflowId);
+			saveSelectedDataset(deletedWorkflow, undefined);
 		} else {
 			const result = await response.json();
 			console.error('Workflow not deleted', result);
@@ -44,7 +45,7 @@
 	}
 
 	/**
-	 * @param {import('$lib/types').Workflow} importedWorkflow
+	 * @param {import('$lib/types-v2').WorkflowV2} importedWorkflow
 	 */
 	function handleWorkflowImported(importedWorkflow) {
 		workflows.push(importedWorkflow);