Improve support for NetCDF fill value and valid range attributes

cypress/e2e/app.cy.ts

@@ -123,21 +123,21 @@ describe('/mock', () => {
     }
   });
 
-  it('visualize datasets with fill value', () => {
-    cy.selectExplorerNode('nD_datasets');
+  it('visualize dataset with fill value', () => {
+    cy.selectExplorerNode('netcdf');
 
-    cy.selectExplorerNode('oneD_fillvalue');
-    cy.findByRole('figure', { name: 'oneD_fillvalue' }).should('be.visible');
+    cy.selectExplorerNode('_FillValue');
+    cy.findByRole('figure', { name: '_FillValue' }).should('be.visible');
 
     if (Cypress.env('TAKE_SNAPSHOTS')) {
-      cy.matchImageSnapshot('fillvalue_1D');
+      cy.matchImageSnapshot('fillvalue_2D');
     }
 
-    cy.selectExplorerNode('twoD_fillvalue');
-    cy.findByRole('figure', { name: 'twoD_fillvalue' }).should('be.visible');
+    cy.selectVisTab('Line');
+    cy.waitForStableDOM();
 
     if (Cypress.env('TAKE_SNAPSHOTS')) {
-      cy.matchImageSnapshot('fillvalue_2D');
+      cy.matchImageSnapshot('fillvalue_1D');
     }
   });
 

packages/app/src/__tests__/NetCDFPack.test.tsx

@@ -0,0 +1,43 @@
+import { screen, within } from '@testing-library/react';
+import { expect, test } from 'vitest';
+
+import { renderApp } from '../test-utils';
+
+test('visualize dataset with `valid_min` and/or `valid_max` attributes', async () => {
+  const { selectExplorerNode } = await renderApp('/netcdf/valid_min');
+
+  const fig1 = screen.getByRole('figure', { name: 'valid_min' });
+  expect(within(fig1).getByText('4e+2')).toBeVisible(); // data max
+  expect(within(fig1).getByText('5e+0')).toBeVisible(); // valid_min
+
+  await selectExplorerNode('valid_max');
+  const fig2 = screen.getByRole('figure', { name: 'valid_max' });
+  expect(within(fig2).getByText('2e+2')).toBeVisible(); // valid_max
+  expect(within(fig2).getByText('−9.5e+1')).toBeVisible(); // data min
+
+  await selectExplorerNode('valid_min_max');
+  const fig3 = screen.getByRole('figure', { name: 'valid_min_max' });
+  expect(within(fig3).getByText('2e+2')).toBeVisible(); // valid_max
+  expect(within(fig3).getByText('5e+0')).toBeVisible(); // valid_min
+});
+
+test('visualize dataset with `valid_range` attribute', async () => {
+  await renderApp('/netcdf/valid_range');
+
+  const fig = screen.getByRole('figure', { name: 'valid_range' });
+  expect(within(fig).getByText('2e+2')).toBeVisible(); // valid_range[1]
+  expect(within(fig).getByText('5e+0')).toBeVisible(); // valid_range[0]
+});
+
+test('visualize dataset with `_FillValue` attribute', async () => {
+  const { selectExplorerNode } = await renderApp('/netcdf/_FillValue');
+
+  const fig1 = screen.getByRole('figure', { name: '_FillValue' });
+  expect(within(fig1).getByText('9.9e+1')).toBeVisible(); // closest data value lower than _FillValue
+  expect(within(fig1).getByText('−9.5e+1')).toBeVisible(); // data min
+
+  await selectExplorerNode('_FillValue (negative)', true);
+  const fig2 = screen.getByRole('figure', { name: '_FillValue (negative)' });
+  expect(within(fig2).getByText('4e+2')).toBeVisible(); // data max
+  expect(within(fig2).getByText('−6e+0')).toBeVisible(); // closest data value greater than _FillValue
+});

packages/app/src/providers/mock/mock-file.ts

@@ -1,6 +1,7 @@
 import { H5T_CSET, H5T_STR } from '@h5web/shared/h5t';
 import { type GroupWithChildren } from '@h5web/shared/hdf5-models';
 import {
+  arrayShape,
   arrayType,
   boolType,
   compoundType,
@@ -116,10 +117,6 @@ export function makeMockFile(): GroupWithChildren {
       group('nD_datasets', [
         array('oneD_linear'),
         array('oneD'),
-        array('oneD_fillvalue', {
-          valueId: 'oneD',
-          attributes: [scalar('_FillValue', 400)],
-        }),
         array('oneD_bigint'),
         array('oneD_cplx'),
         array('oneD_compound', {
@@ -136,10 +133,6 @@ export function makeMockFile(): GroupWithChildren {
           type: enumType(intType(false, 8), ENUM_MAPPING),
         }),
         array('twoD'),
-        array('twoD_fillvalue', {
-          valueId: 'twoD',
-          attributes: [scalar('_FillValue', 400)],
-        }),
         array('twoD_neg'),
         array('twoD_bigint'),
         array('twoD_cplx'),
@@ -435,6 +428,34 @@ export function makeMockFile(): GroupWithChildren {
           ],
         }),
       ]),
+      group('netcdf', [
+        array('valid_min', {
+          valueId: 'twoD',
+          attributes: [scalar('valid_min', 5)],
+        }),
+        array('valid_max', {
+          valueId: 'twoD',
+          attributes: [scalar('valid_max', 200)],
+        }),
+        array('valid_min_max', {
+          valueId: 'twoD',
+          attributes: [scalar('valid_min', 5), scalar('valid_max', 200)],
+        }),
+        array('valid_range', {
+          valueId: 'twoD',
+          attributes: [
+            dataset('valid_range', arrayShape([2]), floatType(), [5, 200]),
+          ],
+        }),
+        array('_FillValue', {
+          valueId: 'twoD',
+          attributes: [scalar('_FillValue', 100)],
+        }),
+        array('_FillValue (negative)', {
+          valueId: 'twoD',
+          attributes: [scalar('_FillValue', -9)],
+        }),
+      ]),
       group('resilience', [
         scalar('error_value', 0, { attributes: [scalarAttr('attr', 1)] }),
         scalar('slow_value', 42),

packages/app/src/test-utils.tsx

@@ -15,7 +15,7 @@ import { type NxDataVis } from './vis-packs/nexus/visualizations';
 
 interface RenderAppResult extends RenderResult {
   user: ReturnType<typeof userEvent.setup>;
-  selectExplorerNode: (name: string) => Promise<void>;
+  selectExplorerNode: (name: string, exact?: boolean) => Promise<void>;
   selectVisTab: (name: Vis | NxDataVis) => Promise<void>;
 }
 
@@ -70,9 +70,11 @@ export async function renderApp(
     user,
     ...renderResult,
 
-    selectExplorerNode: async (name) => {
+    selectExplorerNode: async (name, exact = false) => {
       const item = await screen.findByRole('treeitem', {
-        name: new RegExp(String.raw`^${name}(?: \(NeXus group\))?$`, 'u'), // account for potential NeXus badge
+        name: exact
+          ? name
+          : new RegExp(String.raw`^${name}(?: \(NeXus group\))?$`, 'u'), // account for potential NeXus badge
       });
 
       await user.click(item);

packages/app/src/vis-packs/core/heatmap/HeatmapVisContainer.tsx

@@ -10,8 +10,8 @@ import { useDimMappingState } from '../../../dim-mapping-store';
 import { useValuesInCache } from '../../../hooks';
 import visualizerStyles from '../../../visualizer/Visualizer.module.css';
 import { type VisContainerProps } from '../../models';
+import { useNcIgnoreValue } from '../../netcdf/hooks';
 import VisBoundary from '../../VisBoundary';
-import { useIgnoreFillValue } from '../hooks';
 import ValueFetcher from '../ValueFetcher';
 import { useHeatmapConfig } from './config';
 import MappedHeatmapVis from './MappedHeatmapVis';
@@ -30,9 +30,8 @@ function HeatmapVisContainer(props: VisContainerProps) {
   });
 
   const config = useHeatmapConfig();
-
   const selection = getSliceSelection(dimMapping);
-  const ignoreValue = useIgnoreFillValue(entity);
+  const ignoreValue = useNcIgnoreValue(entity);
 
   return (
     <>

packages/app/src/vis-packs/core/hooks.ts

@@ -10,19 +10,13 @@ import {
   type BuiltInExporter,
   type ExportEntry,
   type ExportFormat,
-  type IgnoreValue,
   type NumArray,
 } from '@h5web/shared/vis-models';
 import { useToggle } from '@react-hookz/web';
 import { type NdArray } from 'ndarray';
 import { useCallback, useMemo } from 'react';
 
 import { useDataContext } from '../../providers/DataProvider';
-import {
-  bigIntTypedArrayFromDType,
-  typedArrayFromDType,
-} from '../../providers/utils';
-import { findScalarNumAttr, getAttributeValue } from '../../utils';
 import { applyMapping, getBaseArray, toNumArray } from './utils';
 
 export const useToNumArray = createMemo(toNumArray);
@@ -97,38 +91,6 @@ export function useMappedArrays(
   );
 }
 
-export function useIgnoreFillValue(dataset: Dataset): IgnoreValue | undefined {
-  const { attrValuesStore } = useDataContext();
-
-  return useMemo(() => {
-    const fillValueAttr = findScalarNumAttr(dataset, '_FillValue');
-    if (!fillValueAttr) {
-      return undefined;
-    }
-
-    const rawFillValue = getAttributeValue(
-      dataset,
-      fillValueAttr,
-      attrValuesStore,
-    );
-
-    const DTypedArray = bigIntTypedArrayFromDType(dataset.type)
-      ? Float64Array // matches `useToNumArray` logic
-      : typedArrayFromDType(dataset.type);
-
-    // Cast fillValue in the type of the dataset values to be able to use `===` for the comparison
-    const fillValue = DTypedArray
-      ? new DTypedArray([Number(rawFillValue)])[0]
-      : undefined;
-
-    if (fillValue === undefined) {
-      return undefined;
-    }
-
-    return (val) => val === fillValue;
-  }, [dataset, attrValuesStore]);
-}
-
 export function useExportEntries<F extends ExportFormat[]>(
   formats: F,
   dataset: Dataset,

packages/app/src/vis-packs/core/line/LineVisContainer.tsx

@@ -9,8 +9,8 @@ import { useDimMappingState } from '../../../dim-mapping-store';
 import { useValuesInCache } from '../../../hooks';
 import visualizerStyles from '../../../visualizer/Visualizer.module.css';
 import { type VisContainerProps } from '../../models';
+import { useNcIgnoreValue } from '../../netcdf/hooks';
 import VisBoundary from '../../VisBoundary';
-import { useIgnoreFillValue } from '../hooks';
 import ValueFetcher from '../ValueFetcher';
 import { useLineConfig } from './config';
 import MappedLineVis from './MappedLineVis';
@@ -28,8 +28,8 @@ function LineVisContainer(props: VisContainerProps) {
   });
 
   const config = useLineConfig();
-  const ignoreValue = useIgnoreFillValue(entity);
   const selection = getSliceSelection(dimMapping);
+  const ignoreValue = useNcIgnoreValue(entity);
 
   return (
     <>

packages/app/src/vis-packs/netcdf/hooks.ts

@@ -0,0 +1,35 @@
+import { type IgnoreValue } from '@h5web/lib';
+import { hasNumericType } from '@h5web/shared/guards';
+import { type ArrayShape, type Dataset } from '@h5web/shared/hdf5-models';
+import { useMemo } from 'react';
+
+import { useDataContext } from '../../providers/DataProvider';
+import { createIgnoreFillValue, getFillValue, getValidRange } from './utils';
+
+/* Priority order: `valid_min` and/or `valid_max`, then `valid_range`, then `_FillValue`.
+ * Note that `_FillValue` acts as an invalid upper or lower bound (if positive or negative respectively).
+ * See NetCDF `valid_range` convention: https://docs.unidata.ucar.edu/netcdf-c/current/attribute_conventions.html */
+export function useNcIgnoreValue(
+  dataset: Dataset<ArrayShape>,
+): IgnoreValue | undefined {
+  const { attrValuesStore } = useDataContext();
+
+  return useMemo(() => {
+    if (!hasNumericType(dataset)) {
+      return undefined;
+    }
+
+    const validRange = getValidRange(dataset, attrValuesStore);
+    if (validRange) {
+      const [validMin, validMax] = validRange;
+      return (val) => val < validMin || val > validMax;
+    }
+
+    const fillValue = getFillValue(dataset, attrValuesStore);
+    if (fillValue !== undefined) {
+      return createIgnoreFillValue(fillValue, dataset.type);
+    }
+
+    return undefined;
+  }, [dataset, attrValuesStore]);
+}

packages/app/src/vis-packs/netcdf/utils.test.ts

@@ -0,0 +1,54 @@
+import { floatType, intType } from '@h5web/shared/hdf5-utils';
+import { describe, expect, it } from 'vitest';
+
+import { createIgnoreFillValue } from './utils';
+
+describe('createIgnoreFillValue', () => {
+  describe('with integers', () => {
+    it('should ignore values greater or equal to positive fill value', () => {
+      const ignoreValue = createIgnoreFillValue(10, intType());
+      expect(ignoreValue(9)).toBe(false);
+      expect(ignoreValue(10)).toBe(true);
+      expect(ignoreValue(11)).toBe(true);
+    });
+
+    it('should ignore values lower or equal to negative fill value', () => {
+      const ignoreValue = createIgnoreFillValue(-10, intType());
+      expect(ignoreValue(-9)).toBe(false);
+      expect(ignoreValue(-10)).toBe(true);
+      expect(ignoreValue(-11)).toBe(true);
+    });
+  });
+
+  describe('with floats', () => {
+    it('should ignore values greater than positive fill value', () => {
+      const ignoreValue = createIgnoreFillValue(0.3, floatType(64));
+      expect(ignoreValue(0.2)).toBe(false);
+      expect(ignoreValue(0.3005)).toBe(true);
+    });
+
+    it('should ignore values close-enough to positive fill value', () => {
+      const ignoreValue = createIgnoreFillValue(0.3, floatType(64));
+      expect(ignoreValue(0.3 - 2 * Number.EPSILON)).toBe(false);
+      expect(ignoreValue(0.3 - Number.EPSILON)).toBe(true);
+      expect(ignoreValue(0.3)).toBe(true);
+      expect(ignoreValue(0.3 + Number.EPSILON)).toBe(true);
+      expect(ignoreValue(0.3 + 2 * Number.EPSILON)).toBe(true);
+    });
+
+    it('should ignore values lower than negative fill value', () => {
+      const ignoreValue = createIgnoreFillValue(-0.3, floatType(64));
+      expect(ignoreValue(-0.2)).toBe(false);
+      expect(ignoreValue(-0.3005)).toBe(true);
+    });
+
+    it('should ignore values close-enough to negative fill value', () => {
+      const ignoreValue = createIgnoreFillValue(-0.3, floatType(64));
+      expect(ignoreValue(-0.3 + 2 * Number.EPSILON)).toBe(false);
+      expect(ignoreValue(-0.3 + Number.EPSILON)).toBe(true);
+      expect(ignoreValue(-0.3)).toBe(true);
+      expect(ignoreValue(-0.3 - Number.EPSILON)).toBe(true);
+      expect(ignoreValue(-0.3 - 2 * Number.EPSILON)).toBe(true);
+    });
+  });
+});