Test HSDS API

.github/workflows/lint-test.yml

@@ -102,12 +102,18 @@ jobs:
         if: steps.poetry-venvs.outputs.cache-hit != 'true'
         run: pnpm support:setup
 
-      - name: Create sample HDF5 file 🎨
-        run: pnpm support:sample
+      - name: Create sample HDF5 files 🎨
+        run: pnpm support:sample && pnpm support:sample --hsds
 
       - name: Start h5grove support server 🔌
         run: pnpm support:h5grove & pnpm wait-on http://localhost:8888 -t 1m
 
+      - name: Start HSDS support server 🔌
+        run: pnpm support:hsds & pnpm wait-on http-get://localhost:5101/?domain=/sample-hsds.h5 -t 1m
+
+      - name: Load HSDS sample file 📥
+        run: pnpm support:hsds:load
+
       - name: Test 👓
         run: pnpm test
 

.gitignore

@@ -7,10 +7,13 @@ dist*/
 /cypress/snapshots/**/__diff_output__
 /cypress/snapshots/**/__received_output__
 
+/support/hsds/root/
+
 .env.local
 .env.*.local
 
 .DS_Store
 Thumbs.db
-*.log
-*.tsbuildinfo
+
+**/*.log
+**/*.tsbuildinfo

CONTRIBUTING.md

@@ -306,8 +306,12 @@ install the recommended extensions.
 - `pnpm test --project <project-name>` - run Vitest on a specific project
 - `pnpm support:setup` - create/update Poetry environments required for
   [testing the providers](#providers-tests)
-- `pnpm support:sample` - create `sample.h5`
+- `pnpm support:sample` - create `sample.h5` (for h5grove and h5wasm)
+- `pnpm support:sample --hsds` - create `sample-hsds.h5` (for HSDS)
 - `pnpm support:h5grove` - start h5grove support server
+- `pnpm support:hsds` - start HSDS support server
+- `pnpm support:hsds:load` - load HSDS sample file (HSDS support server must be
+  running in separate terminal)
 - `pnpm cypress` - open the
   [Cypress](https://docs.cypress.io/guides/overview/why-cypress.html) end-to-end
   test runner (local dev server must be running in separate terminal)
@@ -380,11 +384,11 @@ when calling `pnpm test`.
 
 ### Providers
 
-Two data providers are currently tested through their respective APIs:
-`H5GroveApi` and `H5WasmApi`. Each API test (`<provider>-api.test.ts`) works as
+The data providers are tested through their respective APIs: `H5GroveApi`,
+`H5WasmApi` and `HsdsApi`. Each API test (`<provider>-api.test.ts`) works as
 follows:
 
-1. It instanciates the API using a sample file called `sample.h5`, located in
+1. It instanciates the API using a sample HDF5 file, located in
    `support/sample/dist`, that contains a lot of HDF5 datasets of various shapes
    and types.
 1. It retrieves the values of all the datasets in the sample file and stores
@@ -412,13 +416,24 @@ pyenv and Poetry:
    pyenv exec pnpm support:setup
    ```
 
-Once the Poetry environments are created, you can create `sample.h5`, start
-h5grove and run the API tests:
+Once the Poetry environments are created, create the sample files, start the
+h5grove and HSDS support servers (h5wasm doesn't need one), and run the API
+tests:
 
 ```bash
-pyenv exec pnpm support:sample
+# Create sample files
+pyenv exec pnpm support:sample # for h5grove and h5wasm
+pyenv exec pnpm support:sample --hsds # for HSDS
+
+# Start support servers in separate terminals
 pyenv exec pnpm support:h5grove
-pnpm test api
+pyenv exec pnpm support:hsds
+
+# Load sample file into HSDS
+pyenv exec pnpm support:hsds:load
+
+# Run the API tests
+pnpm test api # or `<provider>-api` for a specific provider's API test
 ```
 
 > If the Python version specified in `.python-version` is globally available on
@@ -440,6 +455,10 @@ environment lacks support for `float128`), you may
 it into the `support/sample/dist` folder. However, please beware that the file
 may not be up to date.
 
+Environment variables defined in `.env.test` files, and overridden in
+`.env.test.local` files, can be used to skip a provider's API test, change its
+support server URL, or change its test file.
+
 ### Visual regression
 
 Cypress is used for end-to-end testing but also for visual regression testing.

package.json

@@ -30,9 +30,11 @@
     "lint:root:eslint": "eslint --max-warnings=0",
     "lint:root:tsc": "tsc",
     "test": "vitest",
-    "support:setup": "poetry -C support/sample install && poetry -C support/h5grove install",
+    "support:setup": "poetry -C support/sample install && poetry -C support/h5grove install && poetry -C support/hsds install",
     "support:sample": "poetry -C support/sample run python create_h5_sample.py",
     "support:h5grove": "poetry -C support/h5grove run python tornado_app.py --basedir ../sample/dist",
+    "support:hsds": "rm -rf support/hsds/hs.log && poetry -C support/hsds run hsds --root_dir . --config_dir ./config --password_file ./config/passwd.txt",
+    "support:hsds:load": "rm -rf support/hsds/root/* && poetry -C support/hsds run hsload -e http://localhost:5101 -u test -p test --link ../sample/dist/sample-hsds.h5 /",
     "cypress": "cypress open --e2e",
     "cypress:run": "cypress run --e2e",
     "version": "pnpm -r sync-version && git add .",

packages/app/.env.test

@@ -3,6 +3,8 @@
 # Copy and configure the variables below in your own `.env.test.local` file
 #####
 
+### h5grove API
+
 # URL of h5grove support server
 VITE_H5GROVE_URL=http://localhost:8888
 
@@ -11,3 +13,14 @@ VITE_H5GROVE_TEST_FILE=sample.h5
 
 # Set to `true` to skip testing h5grove API
 VITE_H5GROVE_SKIP=
+
+### HSDS API
+
+# URL of HSDS support server
+VITE_HSDS_URL=http://localhost:5101
+
+# Name of test domain
+VITE_HSDS_TEST_DOMAIN=/sample-hsds.h5
+
+# Set to `true` to skip testing HSDS API
+VITE_HSDS_SKIP=

packages/app/src/providers/h5grove/h5grove-api.test.ts

@@ -13,16 +13,16 @@ import { H5GroveApi } from './h5grove-api';
 
 const SKIP = import.meta.env.VITE_H5GROVE_SKIP === 'true';
 const H5GROVE_URL = import.meta.env.VITE_H5GROVE_URL;
-const TEST_FILE = import.meta.env.VITE_H5GROVE_TEST_FILE;
+const H5GROVE_TEST_FILE = import.meta.env.VITE_H5GROVE_TEST_FILE;
 assertEnvVar(H5GROVE_URL, 'VITE_H5GROVE_URL');
-assertEnvVar(TEST_FILE, 'VITE_TEST_FILE');
+assertEnvVar(H5GROVE_TEST_FILE, 'VITE_H5GROVE_TEST_FILE');
 
 beforeAll(async () => {
   await assertListeningAt(H5GROVE_URL);
 });
 
 test.skipIf(SKIP)('test file matches snapshot', async () => {
-  const api = new H5GroveApi(H5GROVE_URL, TEST_FILE);
+  const api = new H5GroveApi(H5GROVE_URL, H5GROVE_TEST_FILE);
 
   const root = await api.getEntity('/');
   assertGroup(root);

packages/app/src/providers/h5grove/h5grove-api.ts

@@ -18,7 +18,7 @@ import {
 } from '@h5web/shared/vis-models';
 
 import { DataProviderApi } from '../api';
-import { type Fetcher, type ValuesStoreParams } from '../models';
+import { type ValuesStoreParams } from '../models';
 import { createBasicFetcher, FetcherError, toJSON } from '../utils';
 import {
   type H5GroveAttrValuesResponse,
@@ -38,7 +38,7 @@ export class H5GroveApi extends DataProviderApi {
   public constructor(
     private readonly baseURL: string,
     filepath: string,
-    private readonly fetcher: Fetcher = createBasicFetcher(),
+    private readonly fetcher = createBasicFetcher(),
     private readonly _getExportURL?: DataProviderApi['getExportURL'],
   ) {
     super(filepath);

packages/app/src/providers/hsds/hsds-api.test.ts

@@ -0,0 +1,45 @@
+import {
+  assertDataset,
+  assertEnvVar,
+  assertGroup,
+  assertGroupWithChildren,
+  hasNonNullShape,
+} from '@h5web/shared/guards';
+import { beforeAll, expect, test } from 'vitest';
+
+import { assertListeningAt } from '../../test-utils';
+import { getValueOrError } from '../utils';
+import { HsdsApi } from './hsds-api';
+
+const SKIP = import.meta.env.VITE_HSDS_SKIP === 'true';
+const HSDS_URL = import.meta.env.VITE_HSDS_URL;
+const HSDS_TEST_DOMAIN = import.meta.env.VITE_HSDS_TEST_DOMAIN;
+assertEnvVar(HSDS_URL, 'VITE_HSDS_URL');
+assertEnvVar(HSDS_TEST_DOMAIN, 'VITE_HSDS_TEST_DOMAIN');
+
+beforeAll(async () => {
+  await assertListeningAt(HSDS_URL);
+});
+
+test.skipIf(SKIP)('test file matches snapshot', async () => {
+  const api = new HsdsApi(HSDS_URL, HSDS_TEST_DOMAIN);
+
+  const root = await api.getEntity('/');
+  assertGroup(root);
+  assertGroupWithChildren(root);
+
+  const children = await Promise.all(
+    root.children.map(async (child) => {
+      assertDataset(child);
+      const { name, shape, type, rawType } = child;
+
+      const value = hasNonNullShape(child)
+        ? await getValueOrError(api, child)
+        : null;
+
+      return { name, shape, type, rawType, value };
+    }),
+  );
+
+  expect(children).toMatchSnapshot();
+});

packages/app/src/providers/hsds/hsds-api.ts

@@ -24,8 +24,8 @@ import {
 } from '@h5web/shared/vis-models';
 
 import { DataProviderApi } from '../api';
-import { type Fetcher, type ValuesStoreParams } from '../models';
-import { FetcherError, toJSON } from '../utils';
+import { type ValuesStoreParams } from '../models';
+import { createBasicFetcher, FetcherError, toJSON } from '../utils';
 import {
   type BaseHsdsEntity,
   type HsdsAttribute,
@@ -60,7 +60,7 @@ export class HsdsApi extends DataProviderApi {
   public constructor(
     private readonly baseURL: string,
     filepath: string,
-    private readonly fetcher: Fetcher,
+    private readonly fetcher = createBasicFetcher(),
     private readonly _getExportURL?: DataProviderApi['getExportURL'],
   ) {
     super(filepath);

support/hsds/config/override.yml

@@ -0,0 +1,5 @@
+default_public: True # allow unauthorized read access
+bucket_name: root
+admin_user: test
+max_task_count: 10000 # allow more parallel requests
+log_level: WARN # increased from INFO

support/hsds/config/passwd.txt

@@ -0,0 +1 @@
+test:test