feat: pre-filter workflow sequencing read runs immediately after ENA data is returned (#910) (#926)

* feat: pre-filter workflow sequencing read runs immediately after ENA data is returned (#910)

* feat: make worklow state to preserve stable object (#910)

* feat: pre-filter ena data by taxonomy (#910)

* feat: updated prefiltering based on sequencing step (#910)

---------

Co-authored-by: Fran McDade <[email protected]>

Author: frano-m

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/components/CollectionSelector/collectionSelector.tsx

@@ -8,7 +8,6 @@ import { Table } from "./components/Table/table";
 import { RowSelectionState } from "@tanstack/table-core";
 import { getSequencingData } from "../../utils";
 import { ColumnFilters } from "./components/ColumnFilters/columnFilters";
-import { preSelectColumnFilters } from "./utils";
 
 export const CollectionSelector = ({
   onClose,
@@ -17,15 +16,13 @@ export const CollectionSelector = ({
   selectedCount,
   stepKey,
   table,
-  workflowParameter,
 }: Props): JSX.Element => {
   const [rowSelection, setRowSelection] = useState<RowSelectionState>({});
   return (
     <StyledDialog
       onTransitionEnter={() => {
         setRowSelection(table.getState().rowSelection);
         if (selectedCount > 0) return;
-        preSelectColumnFilters(table, stepKey, workflowParameter);
         table.resetSorting(); // Reset sorting to default.
       }}
       onClose={onClose}

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/components/CollectionSelector/constants.ts

@@ -0,0 +1,10 @@
+import { WORKFLOW_PARAMETER_VARIABLE } from "../../../../../../../../../../../../../../../apis/catalog/brc-analytics-catalog/common/schema-entities";
+import { SEQUENCING_DATA_TYPE } from "../../../../types";
+
+export const WORKFLOW_PARAMETER_BY_STEP_KEY: Record<
+  SEQUENCING_DATA_TYPE.READ_RUNS_PAIRED | SEQUENCING_DATA_TYPE.READ_RUNS_SINGLE,
+  WORKFLOW_PARAMETER_VARIABLE
+> = {
+  readRunsPaired: WORKFLOW_PARAMETER_VARIABLE.SANGER_READ_RUN_PAIRED,
+  readRunsSingle: WORKFLOW_PARAMETER_VARIABLE.SANGER_READ_RUN_SINGLE,
+};

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/components/CollectionSelector/hooks/UseTable/hook.ts

@@ -10,7 +10,7 @@ import { BaseReadRun, ReadRun } from "../../../../types";
 import { ROW_POSITION } from "@databiosphere/findable-ui/lib/components/Table/features/RowPosition/constants";
 import { ROW_PREVIEW } from "@databiosphere/findable-ui/lib/components/Table/features/RowPreview/constants";
 import { columns } from "./columnDef";
-import { useCallback, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { getFacetedUniqueValuesWithArrayValues } from "@databiosphere/findable-ui/lib/components/Table/common/utils";
 import { arrIncludesSome } from "@databiosphere/findable-ui/lib/components/Table/columnDef/columnFilters/filterFn";
 import { ColumnFiltersState, Updater } from "@tanstack/react-table";
@@ -43,6 +43,17 @@ export const useTable = (
     [ENA_QUERY_METHOD.TAXONOMY_ID]: [],
   });
 
+  // Grab the column filters for ENA by taxonomy ID.
+  const { columnFilters } = enaTaxonomyId;
+
+  useEffect(() => {
+    if (columnFilters.length === 0) return;
+    // Pre-filter the table data for ENA by taxonomy ID.
+    setColumnFiltersByMethod(
+      updateColumnFilters(ENA_QUERY_METHOD.TAXONOMY_ID, columnFilters)
+    );
+  }, [columnFilters]);
+
   const onColumnFiltersChange = useCallback(
     (updaterOrValue: Updater<ColumnFiltersState>): void =>
       setColumnFiltersByMethod(

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/components/CollectionSelector/types.ts

@@ -2,7 +2,6 @@ import { ReadRun } from "../../types";
 import { OnConfigure } from "../../../../../../../../../../../../../../../views/WorkflowInputsView/hooks/UseConfigureInputs/types";
 import { Table } from "@tanstack/react-table";
 import { SEQUENCING_DATA_TYPE } from "../../../../types";
-import { WorkflowParameter } from "app/apis/catalog/brc-analytics-catalog/common/entities";
 
 export interface Props {
   onClose: () => void;
@@ -11,5 +10,4 @@ export interface Props {
   selectedCount: number;
   stepKey: SEQUENCING_DATA_TYPE;
   table: Table<ReadRun>;
-  workflowParameter?: WorkflowParameter;
 }

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/components/CollectionSelector/utils.ts

@@ -1,27 +1,28 @@
-import { ENA_QUERY_METHOD, SEQUENCING_DATA_TYPE } from "../../../../types";
-import { ColumnFiltersState, Table } from "@tanstack/react-table";
-import { ReadRun } from "../../types";
+import { SEQUENCING_DATA_TYPE } from "../../../../types";
+import { ColumnFiltersState } from "@tanstack/react-table";
 import { CATEGORY_CONFIGS } from "./hooks/UseTable/categoryConfigs";
-import { WorkflowParameter } from "app/apis/catalog/brc-analytics-catalog/common/entities";
+import {
+  Workflow,
+  WorkflowParameter,
+} from "../../../../../../../../../../../../../../../apis/catalog/brc-analytics-catalog/common/entities";
+import { WORKFLOW_PARAMETER_BY_STEP_KEY } from "./constants";
 
 /**
  * Builds column filters from preselected values only if they are valid for the current data.
- * A set of preselected filters is considered valid if at least one row in the table satisfies all column filters (AND across columns),
+ * A set of preselected filters is considered valid if at least one row of data satisfies all column filters (AND across columns),
  * where each column filter matches if the row contains at least one of the desired values for that column (OR within a column).
  * If no rows satisfy the filters, an empty list is returned.
  *
  * Note: This does not mutate or filter out individual values; it validates the combination overall.
  *
- * @param table - Table instance.
+ * @param rows - Rows (read runs).
  * @param preselectedFilters - Record of column IDs to arrays of desired filter values.
  * @returns Column filters state.
  */
-function buildValidatedColumnFilters(
-  table: Table<ReadRun>,
+function buildValidatedColumnFilters<T>(
+  rows: T[],
   preselectedFilters: Record<string, string[]>
 ): ColumnFiltersState {
-  const { rows } = table.getRowModel();
-
   // We need to ensure that the combined column filters are valid for the given data.
   // A row must have at least one of the desired values (OR-join) for each column filter (AND-join).
   for (const row of rows) {
@@ -32,7 +33,7 @@ function buildValidatedColumnFilters(
       const filterValueSet = new Set(value as string[]);
 
       // Determine the column values.
-      const columnValues = toStringArray(row.getValue(id));
+      const columnValues = toStringArray(row[id as keyof T]);
 
       // At least one of the desired values must be present in the column "OR".
       let hasAny = false;
@@ -54,7 +55,7 @@ function buildValidatedColumnFilters(
     if (!isRowValid) continue;
 
     // Found a valid row i.e. we know the pre-selected column filters will be a valid
-    // combination for the table data. There is no need to continue checking the rest of the rows.
+    // combination for the data. There is no need to continue checking the rest of the rows.
     return Object.entries(preselectedFilters).map(([id, value]) => ({
       id,
       value,
@@ -65,18 +66,6 @@ function buildValidatedColumnFilters(
   return [];
 }
 
-/**
- * Returns true if the table is using data from ENA query method by taxonomy ID.
- * @param table - The table.
- * @returns True if the table is using data from ENA query method by taxonomy ID.
- */
-function isENAQueryMethodByTaxonomyId(table: Table<ReadRun>): boolean {
-  if (!table.options.meta) return false;
-  if (!("enaQueryMethod" in table.options.meta)) return false;
-
-  return table.options.meta.enaQueryMethod === ENA_QUERY_METHOD.TAXONOMY_ID;
-}
-
 /**
  * Gets library strategy requirements from a workflow parameter.
  * @param parameter - The workflow parameter.
@@ -131,14 +120,33 @@ export function getDescriptionRequirement(
   return parameter.data_requirements.description;
 }
 
+/**
+ * Returns the workflow parameter for the given step key.
+ * @param workflow - Workflow.
+ * @param stepKey - Step key.
+ * @returns Workflow parameter for the given step key.
+ */
+export function getWorkflowParameter(
+  workflow: Workflow,
+  stepKey:
+    | SEQUENCING_DATA_TYPE.READ_RUNS_PAIRED
+    | SEQUENCING_DATA_TYPE.READ_RUNS_SINGLE
+): WorkflowParameter | undefined {
+  return workflow.parameters.find(
+    ({ variable }) => variable === WORKFLOW_PARAMETER_BY_STEP_KEY[stepKey]
+  );
+}
+
 /**
  * Builds filters based on step key and workflow parameter requirements.
  * @param stepKey - The sequencing data type (paired or single).
  * @param workflowParameter - Optional workflow parameter with data requirements.
  * @returns Filters with layout, strategy, source, and other requirements.
  */
 export function buildFilters(
-  stepKey: SEQUENCING_DATA_TYPE,
+  stepKey:
+    | SEQUENCING_DATA_TYPE.READ_RUNS_PAIRED
+    | SEQUENCING_DATA_TYPE.READ_RUNS_SINGLE,
   workflowParameter?: WorkflowParameter
 ): Record<string, string[]> {
   const filters: Record<string, string[]> = {};
@@ -186,22 +194,21 @@ export function buildFilters(
 
 /**
  * Pre-selects column filters based on the given step key for ENA query method by taxonomy ID.
- * @param table - Table instance.
+ * @param workflow - Workflow.
+ * @param rows - Rows (read runs).
  * @param stepKey - Step key.
- * @param workflowParameter - Optional workflow parameter with data requirements.
+ * @returns Column filters for the given ENA data.
  */
-export function preSelectColumnFilters(
-  table: Table<ReadRun>,
-  stepKey: SEQUENCING_DATA_TYPE,
-  workflowParameter?: WorkflowParameter
-): void {
-  // Do not apply any filters for step key READ_RUNS_ANY.
-  if (stepKey === SEQUENCING_DATA_TYPE.READ_RUNS_ANY) return;
-  if (isENAQueryMethodByTaxonomyId(table)) {
-    const filters = buildFilters(stepKey, workflowParameter);
-    const columnFiltersState = buildValidatedColumnFilters(table, filters);
-    table.setColumnFilters(columnFiltersState);
-  }
+export function preSelectColumnFilters<T>(
+  workflow: Workflow,
+  rows: T[],
+  stepKey:
+    | SEQUENCING_DATA_TYPE.READ_RUNS_PAIRED
+    | SEQUENCING_DATA_TYPE.READ_RUNS_SINGLE
+): ColumnFiltersState {
+  const workflowParameter = getWorkflowParameter(workflow, stepKey);
+  const filters = buildFilters(stepKey, workflowParameter);
+  return buildValidatedColumnFilters(rows, filters);
 }
 
 /**

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/enaSequencingData.tsx

@@ -15,7 +15,6 @@ export const ENASequencingData = ({
   stepKey,
   table,
   taxonomicLevelSpecies,
-  workflowParameter,
 }: Props): JSX.Element => {
   const accessionDialog = useDialog();
   const collectionDialog = useDialog();
@@ -53,7 +52,6 @@ export const ENASequencingData = ({
         selectedCount={selectedCount}
         table={table}
         stepKey={stepKey}
-        workflowParameter={workflowParameter}
       />
       <CollectionSummary
         onClear={() => {

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/hooks/UseENADataByTaxonomyId/hook.ts

@@ -3,18 +3,28 @@ import { UseENADataByTaxonomyId } from "./types";
 import { fetchENAData } from "./request";
 import { useAsync } from "@databiosphere/findable-ui/lib/hooks/useAsync";
 import { shouldFetch } from "./utils";
-import { BRCDataCatalogGenome } from "../../../../../../../../../../../../../../../apis/catalog/brc-analytics-catalog/common/entities";
+import {
+  BRCDataCatalogGenome,
+  Workflow,
+} from "../../../../../../../../../../../../../../../apis/catalog/brc-analytics-catalog/common/entities";
 import { GA2AssemblyEntity } from "../../../../../../../../../../../../../../../apis/catalog/ga2/entities";
 import { config } from "../../../../../../../../../../../../../../../../app/config/config";
+import { ColumnFiltersState } from "@tanstack/react-table";
+import { preSelectColumnFilters } from "../../components/CollectionSelector/utils";
+import { SEQUENCING_DATA_TYPE } from "../../../../types";
 
 export const useENADataByTaxonomyId = <T>(
-  genome: BRCDataCatalogGenome | GA2AssemblyEntity
+  workflow: Workflow,
+  genome: BRCDataCatalogGenome | GA2AssemblyEntity,
+  stepKey: SEQUENCING_DATA_TYPE
 ): UseENADataByTaxonomyId<T> => {
   const { data, run } = useAsync<T[] | undefined>();
+  const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const [errors, setErrors] = useState<Record<string, string>>({});
   const [loading, setLoading] = useState<boolean>(true);
   const { ncbiTaxonomyId: taxonomyId } = genome;
   const { maxReadRunsForBrowseAll } = config();
+
   const onRequestData = useCallback(async (): Promise<void> => {
     run(
       fetchENAData({
@@ -23,14 +33,19 @@ export const useENADataByTaxonomyId = <T>(
             setErrors({ taxonomyId: e.message });
             setLoading(false);
           },
-          onSuccess: () => {
+          onSuccess: (readRuns) => {
             setLoading(false);
+            // If the step key is READ_RUNS_ANY, do not pre-select column filters.
+            if (stepKey === SEQUENCING_DATA_TYPE.READ_RUNS_ANY) return;
+            setColumnFilters(
+              preSelectColumnFilters(workflow, readRuns, stepKey)
+            );
           },
         },
         taxonomyId,
       })
     );
-  }, [run, taxonomyId]);
+  }, [run, stepKey, taxonomyId, workflow]);
 
   useEffect(() => {
     if (shouldFetch(taxonomyId, maxReadRunsForBrowseAll)) {
@@ -43,5 +58,5 @@ export const useENADataByTaxonomyId = <T>(
     }
   }, [onRequestData, taxonomyId, maxReadRunsForBrowseAll]);
 
-  return { data, status: { errors, loading } };
+  return { columnFilters, data, status: { errors, loading } };
 };

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/hooks/UseENADataByTaxonomyId/request.ts

@@ -13,7 +13,7 @@ export async function fetchENAData<T>({
   submitOptions,
   taxonomyId,
 }: {
-  submitOptions: Required<SubmitOptions>;
+  submitOptions: Required<SubmitOptions<T>>;
   taxonomyId: string;
 }): Promise<T[] | undefined> {
   try {
@@ -36,7 +36,7 @@ export async function fetchENAData<T>({
       return;
     }
 
-    submitOptions.onSuccess?.();
+    submitOptions.onSuccess?.(data);
 
     return data;
   } catch (error) {

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/hooks/UseENADataByTaxonomyId/types.ts

@@ -1,19 +1,22 @@
-export type OnRequestData = (
+import { ColumnFiltersState } from "@tanstack/react-table";
+
+export type OnRequestData<T> = (
   taxonomyId: string,
-  submitOptions?: SubmitOptions
+  submitOptions?: SubmitOptions<T>
 ) => Promise<void>;
 
 export interface Status {
   errors: Record<string, string>;
   loading: boolean;
 }
 
-export interface SubmitOptions {
+export interface SubmitOptions<T> {
   onError?: (error: Error) => void;
-  onSuccess?: () => void;
+  onSuccess?: (data: T[]) => void;
 }
 
 export interface UseENADataByTaxonomyId<T> {
+  columnFilters: ColumnFiltersState;
   data?: T[];
   status: Status;
 }

app/components/Entity/components/ConfigureWorkflowInputs/components/Main/components/Stepper/components/Step/SequencingStep/components/ENASequencingData/types.ts

@@ -5,7 +5,6 @@ import { SEQUENCING_DATA_TYPE } from "../../types";
 import { UseENADataByTaxonomyId } from "./hooks/UseENADataByTaxonomyId/types";
 import { Dispatch, SetStateAction } from "react";
 import { ENA_QUERY_METHOD } from "../../types";
-import { WorkflowParameter } from "app/apis/catalog/brc-analytics-catalog/common/entities";
 
 export interface BaseReadRun {
   base_count: number;
@@ -44,5 +43,4 @@ export interface Props {
   stepKey: SEQUENCING_DATA_TYPE;
   table: Table<ReadRun>;
   taxonomicLevelSpecies: string;
-  workflowParameter?: WorkflowParameter;
 }