Support 64-bit integers

Author: axelboc

packages/app/src/metadata-viewer/utils.ts

@@ -1,14 +1,19 @@
 import {
+  isBigIntegerType,
+  isFloatType,
   isH5WebComplex,
   isIntegerType,
   isNumericType,
   isScalarShape,
 } from '@h5web/shared/guards';
 import {
+  type BigIntegerType,
   type ComplexArray,
   type DType,
   DTypeClass,
   type H5WebComplex,
+  type IntegerType,
+  type NumericType,
   type Shape,
 } from '@h5web/shared/hdf5-models';
 import { formatScalarComplex } from '@h5web/shared/vis-utils';
@@ -27,18 +32,21 @@ export function renderShape(shape: Shape): string {
     : `${shape.join(' x ')} = ${shape.reduce((acc, value) => acc * value)}`;
 }
 
-export function renderType(type: DType): string {
-  if (isNumericType(type)) {
-    const { endianness, size } = type;
+function renderIntSign(type: IntegerType | BigIntegerType): string {
+  return type.signed ? ' (signed)' : ' (unsigned)';
+}
+
+function renderEndianness(type: NumericType | BigIntegerType): string {
+  return type.endianness ? `, ${type.endianness}` : '';
+}
 
-    const endiannessStr = endianness ? `, ${endianness}` : '';
-    const signStr = isIntegerType(type)
-      ? type.signed
-        ? ' (signed)'
-        : ' (unsigned)'
-      : '';
+export function renderType(type: DType): string {
+  if (isIntegerType(type) || isBigIntegerType(type)) {
+    return `Integer${renderIntSign(type)}, ${type.size}-bit${renderEndianness(type)}`;
+  }
 
-    return `${type.class}${signStr}, ${size}-bit${endiannessStr}`;
+  if (isFloatType(type)) {
+    return `Float, ${type.size}-bit${renderEndianness(type)}`;
   }
 
   if (type.class === DTypeClass.String) {

packages/app/src/providers/mock/mock-file.ts

@@ -2,6 +2,7 @@ import { H5T_CSET, H5T_STR } from '@h5web/shared/h5t';
 import { type GroupWithChildren } from '@h5web/shared/hdf5-models';
 import {
   arrayType,
+  bigintType,
   boolType,
   compoundType,
   cplx,
@@ -152,6 +153,7 @@ export function makeMockFile(): GroupWithChildren {
       group('typed_arrays', [
         array('uint8', { type: intType(false, 8) }),
         array('int16', { type: intType(true, 16) }),
+        array('int64', { type: bigintType() }),
         array('float32', { type: floatType(32) }),
         array('float64', { type: floatType(64) }),
         withImageAttr(array('uint8_rgb', { type: intType(false, 8) })),

packages/app/src/vis-packs/core/matrix/utils.ts

@@ -65,7 +65,7 @@ export function getFormatter(
     return createEnumFormatter(type.mapping);
   }
 
-  return (val) => (val as string).toString(); // call `toString()` for safety, in case type cast is wrong
+  return (val) => (val as string | bigint).toString(); // call `toString()` for safety, in case type cast is wrong
 }
 
 export function getCellWidth(

packages/app/src/vis-packs/core/scalar/utils.ts

@@ -27,5 +27,5 @@ export function getFormatter(
     return createEnumFormatter(dataset.type.mapping);
   }
 
-  return (val) => (val as number | string).toString();
+  return (val) => (val as number | bigint | string).toString();
 }

packages/app/src/vis-packs/core/utils.ts

@@ -1,5 +1,10 @@
 import { type InteractionInfo } from '@h5web/lib';
-import { isIntegerType, isNumericType } from '@h5web/shared/guards';
+import {
+  isBigIntegerType,
+  isBigIntTypeArray,
+  isIntegerType,
+  isNumericType,
+} from '@h5web/shared/guards';
 import {
   type ArrayValue,
   DTypeClass,
@@ -93,11 +98,23 @@ export function getImageInteractions(keepRatio: boolean): InteractionInfo[] {
   return keepRatio ? BASE_INTERACTIONS : INTERACTIONS_WITH_AXIAL_ZOOM;
 }
 
+function isBigIntArray(val: ArrayValue<NumericLikeType>): val is bigint[] {
+  return Array.isArray(val) && typeof val[0] === 'bigint';
+}
+
 function isBoolArray(val: ArrayValue<NumericLikeType>): val is boolean[] {
   return Array.isArray(val) && typeof val[0] === 'boolean';
 }
 
 export function toNumArray(arr: ArrayValue<NumericLikeType>): NumArray {
+  if (isBigIntTypeArray(arr)) {
+    return Float64Array.from(arr, Number); // cast to float 64
+  }
+
+  if (isBigIntArray(arr)) {
+    return arr.map(Number); // cast to float 64
+  }
+
   if (isBoolArray(arr)) {
     return arr.map((val) => (val ? 1 : 0));
   }
@@ -109,11 +126,12 @@ const TYPE_STRINGS: Record<NumericLikeType['class'], string> = {
   [DTypeClass.Bool]: 'bool',
   [DTypeClass.Enum]: 'enum',
   [DTypeClass.Integer]: 'int',
+  [DTypeClass.BigInteger]: 'int',
   [DTypeClass.Float]: 'float',
 };
 
 export function formatNumLikeType(type: NumericLikeType): string {
-  const unsignedPrefix = isIntegerType(type) && !type.signed ? 'u' : '';
-  const sizeSuffix = isNumericType(type) ? type.size : '';
+  const unsignedPrefix = 'signed' in type && !type.signed ? 'u' : '';
+  const sizeSuffix = 'size' in type ? type.size : '';
   return `${unsignedPrefix}${TYPE_STRINGS[type.class]}${sizeSuffix}`;
 }

packages/h5wasm/src/h5wasm-api.ts

@@ -16,12 +16,7 @@ import {
 import { transfer } from 'comlink';
 
 import { type Plugin } from './models';
-import {
-  getEnhancedError,
-  hasBigInts,
-  PLUGINS_BY_FILTER_ID,
-  sanitizeBigInts,
-} from './utils';
+import { getEnhancedError, PLUGINS_BY_FILTER_ID } from './utils';
 import { type H5WasmWorkerAPI } from './worker';
 
 export class H5WasmApi extends DataProviderApi {
@@ -48,8 +43,7 @@ export class H5WasmApi extends DataProviderApi {
     await this.processFilters(dataset);
 
     try {
-      const value = await this.remote.getValue(fileId, dataset.path, selection);
-      return hasBigInts(dataset.type) ? sanitizeBigInts(value) : value;
+      return await this.remote.getValue(fileId, dataset.path, selection);
     } catch (error) {
       throw getEnhancedError(error);
     }

packages/h5wasm/src/utils.ts

@@ -1,5 +1,3 @@
-import { type DType, DTypeClass } from '@h5web/shared/hdf5-models';
-
 import { type HDF5Diag, Plugin } from './models';
 
 // https://support.hdfgroup.org/services/contributions.html
@@ -15,38 +13,6 @@ export const PLUGINS_BY_FILTER_ID: Record<number, Plugin> = {
   32_026: Plugin.Blosc2,
 };
 
-export function hasBigInts(type: DType): boolean {
-  if (
-    type.class === DTypeClass.Enum ||
-    type.class === DTypeClass.Array ||
-    type.class === DTypeClass.VLen
-  ) {
-    return hasBigInts(type.base);
-  }
-
-  if (type.class === DTypeClass.Compound) {
-    return Object.values(type.fields).some(hasBigInts);
-  }
-
-  return type.class === DTypeClass.Integer && type.size === 64;
-}
-
-export function sanitizeBigInts(value: unknown): unknown {
-  if (Array.isArray(value)) {
-    return value.map(sanitizeBigInts);
-  }
-
-  if (value instanceof BigInt64Array || value instanceof BigUint64Array) {
-    return [...value].map(Number);
-  }
-
-  if (typeof value === 'bigint') {
-    return Number(value);
-  }
-
-  return value;
-}
-
 const DIAG_PREDICATES: ((diag: HDF5Diag) => boolean)[] = [
   (diag: HDF5Diag) => {
     return (

packages/h5wasm/src/worker-utils.ts

@@ -21,7 +21,7 @@ import {
   enumOrBoolType,
   floatType,
   getNameFromPath,
-  intType,
+  intOrBigintType,
   opaqueType,
   referenceType,
   strType,
@@ -216,12 +216,13 @@ function parseDType(metadata: Metadata): DType {
 
   if (h5tClass === H5T_CLASS.INTEGER) {
     const { signed, littleEndian } = metadata;
-    return intType(
+    return intOrBigintType(
       signed,
       size * 8,
       littleEndian ? H5T_ORDER.LE : H5T_ORDER.BE,
     );
   }
+
   if (h5tClass === H5T_CLASS.FLOAT) {
     const { littleEndian } = metadata;
     return floatType(size * 8, littleEndian ? H5T_ORDER.LE : H5T_ORDER.BE);

packages/shared/src/guards.ts

@@ -2,6 +2,7 @@ import { type Data, type NdArray, type TypedArray } from 'ndarray';
 
 import {
   type ArrayShape,
+  type BigIntegerType,
   type BooleanType,
   type ComplexArray,
   type ComplexType,
@@ -30,20 +31,12 @@ import {
 import {
   type AnyNumArray,
   type AxisScaleType,
+  type BigIntTypedArray,
   type ColorScaleType,
   type NumArray,
 } from './vis-models';
 import { AXIS_SCALE_TYPES, COLOR_SCALE_TYPES, getValues } from './vis-utils';
 
-const PRINTABLE_DTYPES = new Set([
-  DTypeClass.Integer,
-  DTypeClass.Float,
-  DTypeClass.String,
-  DTypeClass.Bool,
-  DTypeClass.Enum,
-  DTypeClass.Complex,
-]);
-
 export function isAbsolutePath(path: string): boolean {
   return path.startsWith('/');
 }
@@ -86,6 +79,12 @@ function assertNum(val: unknown): asserts val is number {
   }
 }
 
+function assertNumOrBigint(val: unknown): asserts val is number {
+  if (typeof val !== 'number' && typeof val !== 'bigint') {
+    throw new TypeError('Expected number');
+  }
+}
+
 function assertNumOrBool(val: unknown): asserts val is number | boolean {
   if (typeof val !== 'number' && typeof val !== 'boolean') {
     throw new TypeError('Expected boolean or number');
@@ -142,6 +141,10 @@ export function isTypedArray(val: unknown): val is TypedArray {
   );
 }
 
+export function isBigIntTypeArray(val: unknown): val is BigIntTypedArray {
+  return val instanceof BigInt64Array || val instanceof BigUint64Array;
+}
+
 export function isGroup(entity: Entity): entity is Group {
   return entity.kind === EntityKind.Group;
 }
@@ -313,11 +316,23 @@ export function assertNumericType<S extends Shape>(
   }
 }
 
+export function isBigIntegerType(type: DType): type is BigIntegerType {
+  return type.class === DTypeClass.BigInteger;
+}
+
+export function isNumericLikeType(type: DType): type is NumericLikeType {
+  return (
+    isNumericType(type) ||
+    isBigIntegerType(type) ||
+    isBoolType(type) ||
+    isEnumType(type)
+  );
+}
+
 export function hasNumericLikeType<S extends Shape>(
   dataset: Dataset<S>,
 ): dataset is Dataset<S, NumericLikeType> {
-  const { type } = dataset;
-  return isNumericType(type) || isBoolType(type) || isEnumType(type);
+  return isNumericLikeType(dataset.type);
 }
 
 export function assertNumericLikeType<S extends Shape>(
@@ -360,22 +375,20 @@ export function assertNumericLikeOrComplexType<S extends Shape>(
   }
 }
 
+export function isPrintableType(type: DType): type is PrintableType {
+  return isStringType(type) || isNumericLikeType(type) || isComplexType(type);
+}
+
 export function hasPrintableType<S extends Shape>(
-  entity: Dataset<S>,
-): entity is Dataset<S, PrintableType> {
-  return PRINTABLE_DTYPES.has(entity.type.class);
+  dataset: Dataset<S>,
+): dataset is Dataset<S, PrintableType> {
+  return isPrintableType(dataset.type);
 }
 
 export function assertPrintableType<S extends Shape>(
   dataset: Dataset<S>,
 ): asserts dataset is Dataset<S, PrintableType> {
-  if (
-    !hasStringType(dataset) &&
-    !hasNumericType(dataset) &&
-    !hasBoolType(dataset) &&
-    !hasEnumType(dataset) &&
-    !hasComplexType(dataset)
-  ) {
+  if (!hasPrintableType(dataset)) {
     throw new Error('Expected dataset to have displayable type');
   }
 }
@@ -402,7 +415,7 @@ export function hasPrintableCompoundType<S extends Shape>(
   dataset: Dataset<S, CompoundType>,
 ): dataset is Dataset<S, CompoundType<PrintableType>> {
   const { fields } = dataset.type;
-  return Object.values(fields).every((f) => PRINTABLE_DTYPES.has(f.class));
+  return Object.values(fields).every(isPrintableType);
 }
 
 export function assertPrintableCompoundType<S extends Shape>(
@@ -420,22 +433,24 @@ export function isComplexValue(
   return type.class === DTypeClass.Complex;
 }
 
-function assertPrimitiveValue(
+function assertScalarValue(
   type: DType,
   value: unknown,
 ): asserts value is ScalarValue {
   if (isNumericType(type)) {
     assertNum(value);
   } else if (isStringType(type)) {
     assertStr(value);
+  } else if (isBigIntegerType(type)) {
+    assertNumOrBigint(value);
   } else if (isBoolType(type)) {
     assertNumOrBool(value);
   } else if (isComplexType(type)) {
     assertComplex(value);
   } else if (isCompoundType(type)) {
     assertArray(value);
     Object.values(type.fields).forEach((fieldType, index) => {
-      assertPrimitiveValue(fieldType, value[index]);
+      assertScalarValue(fieldType, value[index]);
     });
   }
 }
@@ -447,14 +462,18 @@ export function assertDatasetValue<D extends Dataset<ScalarShape | ArrayShape>>(
   const { type } = dataset;
 
   if (hasScalarShape(dataset)) {
-    assertPrimitiveValue(type, value);
+    assertScalarValue(type, value);
   } else {
-    if (!Array.isArray(value) && !isTypedArray(value)) {
+    if (
+      !Array.isArray(value) &&
+      !isTypedArray(value) &&
+      !isBigIntTypeArray(value)
+    ) {
       throw new TypeError('Expected array or typed array');
     }
 
     if (value.length > 0) {
-      assertPrimitiveValue(type, value[0]);
+      assertScalarValue(type, value[0]);
     }
   }
 }