diff --git a/CHANGELOG.md b/CHANGELOG.md index b2e7a3f4..93fdd8e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ # Changes +## June 9, 2025 +- Added DownsampledTimeseriesClient with downsampling methods LTTB (Largest Triangle Three Buckets) and Decimate for improved performance with large datasets + +## June 6, 2025 +- Added SequentialRecordingsTable plugin for NWB files to visualize SequentialRecordingsTable neurodata type + ## June 5, 2025 - Modernized Python package structure with pyproject.toml configuration. Removed legacy setup.py, setup.cfg, and setup.cfg.j2 files - Added option for using local neurosift server with the CLI +- Fixed CORS policy in local file access server to allow any localhost port for development ## May 20, 2025 - Added support for resolving NWB file URLs from dandiset path in NwbPage diff --git a/docs/llm_docs/nwb_read_in_neurosift.md b/docs/llm_docs/nwb_read_in_neurosift.md new file mode 100644 index 00000000..3347a3ca --- /dev/null +++ b/docs/llm_docs/nwb_read_in_neurosift.md @@ -0,0 +1,220 @@ +# NWB Data Reading in Neurosift + +This document explains how Neurosift reads and processes NWB (Neurodata Without Borders) files, including the architecture, optimizations, and technical details relevant for contributors. + +## Overview + +Neurosift uses a multi-layered approach to read NWB files efficiently in the browser. The system supports both traditional HDF5 files and optimized LINDI (Linked Data Interface) files, with intelligent format detection and performance optimizations. + +## Architecture Components + +### 1. Entry Point (`src/pages/NwbPage/NwbPage.tsx`) +- Handles URL processing and format detection +- Manages DANDI API integration for asset resolution +- Coordinates LINDI optimization attempts + +### 2. HDF5 Interface Layer (`src/pages/NwbPage/hdf5Interface.ts`) +- Central abstraction for all file operations +- Implements caching and request deduplication +- Manages authentication and error handling +- Provides React hooks for component integration + +### 3. Remote File Access (`src/remote-h5-file/`) +- Core file reading implementation +- Supports multiple file formats (HDF5, LINDI) +- Handles HTTP range requests and chunking +- Web Worker integration for non-blocking operations + +## Data Flow + +``` +URL Input → Format Detection → LINDI Check → File Access → Caching → Visualization + ↓ ↓ ↓ ↓ ↓ ↓ +NwbPage.tsx → hdf5Interface → tryGetLindiUrl → RemoteH5File* → Cache → Plugins +``` + +### Step-by-Step Process + +1. **URL Resolution**: Convert DANDI paths to direct download URLs +2. **LINDI Detection**: Check for optimized `.lindi.json` or `.lindi.tar` files +3. **File Access**: Use appropriate reader (HDF5 or LINDI) +4. **Data Loading**: Lazy load only required data with chunking +5. **Caching**: Store results to avoid redundant requests +6. **Visualization**: Pass data to type-specific plugins + +## File Format Support + +### Traditional HDF5 Files +- **Access Method**: HTTP range requests via Web Workers +- **Worker URL**: `https://tempory.net/js/RemoteH5Worker.js` +- **Chunk Size**: 100KB default (configurable) +- **Limitations**: Slower metadata access, requires full header parsing + +### LINDI Files (Optimized) +- **Format**: JSON-based reference file system +- **Metadata**: Instant access to all HDF5 structure +- **Data Storage**: References to external URLs or embedded chunks +- **Location**: `https://lindi.neurosift.org/[dandi|dandi-staging]/dandisets/{id}/assets/{asset_id}/nwb.lindi.json` +- **Tar Support**: `.lindi.tar` files containing both metadata and data + +## Performance Optimizations + +### 1. LINDI Priority System +```typescript +if (isDandiAssetUrl(url) && currentDandisetId && tryUsingLindi) { + const lindiUrl = await tryGetLindiUrl(url, currentDandisetId); + if (lindiUrl) return { url: lindiUrl }; // 10-100x faster metadata access +} +``` + +### 2. Lazy Loading Strategy +- **Groups**: Load structure on-demand +- **Datasets**: Load metadata separately from data +- **Data**: Only load when visualization requires it + +### 3. HTTP Range Requests +- Load only required byte ranges from large files +- Configurable chunk sizes for optimal network usage +- Automatic retry logic for failed requests + +### 4. Multi-Level Caching +- **In-Memory**: Groups, datasets, and data results +- **Request Deduplication**: Prevent duplicate network calls +- **Status Tracking**: Monitor ongoing operations + +### 5. Web Workers +- Non-blocking file operations +- Prevents UI freezing during large data loads +- Single worker by default (configurable) + +## Technical Limits and Constraints + +### Data Size Limits +```typescript +const maxNumElements = 1e7; // 10 million elements maximum +if (totalSize > maxNumElements) { + throw new Error(`Dataset too large: ${formatSize(totalSize)} > ${formatSize(maxNumElements)}`); +} +``` + +### Slicing Constraints +- Maximum 3 dimensions can be sliced simultaneously +- Slice parameters must be valid integers +- Format: `[[start, end], [start, end], ...]` + +### Authentication Requirements +- DANDI API key required for embargoed datasets +- Automatic detection of authentication errors +- User notification system for access issues + +## Key Implementation Details + +### Core Functions + +#### `getHdf5Group(url, path)` +- Returns HDF5 group structure with subgroups and datasets +- Implements caching to avoid redundant requests +- Used for building file hierarchy views + +#### `getHdf5Dataset(url, path)` +- Returns dataset metadata (shape, dtype, attributes) +- Does not load actual data +- Essential for understanding data structure before loading + +#### `getHdf5DatasetData(url, path, options)` +- Loads actual array data with optional slicing +- Supports cancellation via `Canceler` objects +- Handles BigInt conversion for compatibility + +### React Integration +```typescript +// Hook-based API for components +const group = useHdf5Group(url, "/acquisition"); +const dataset = useHdf5Dataset(url, "/data/timeseries"); +const { data, errorMessage } = useHdf5DatasetData(url, "/data/values"); +``` + +### Error Handling +- Network timeout handling (3-minute default) +- Authentication error detection and user notification +- Graceful fallbacks for failed LINDI attempts +- CORS issue mitigation strategies + +## DANDI Integration + +### Asset URL Resolution +```typescript +// Convert DANDI paths to download URLs +const response = await fetch( + `https://api.dandiarchive.org/api/dandisets/${dandisetId}/versions/${version}/assets/?glob=${path}` +); +const assetId = data.results[0].asset_id; +const downloadUrl = `https://api.dandiarchive.org/api/assets/${assetId}/download/`; +``` + +### LINDI URL Construction +```typescript +const aa = staging ? "dandi-staging" : "dandi"; +const lindiUrl = `https://lindi.neurosift.org/${aa}/dandisets/${dandisetId}/assets/${assetId}/nwb.lindi.json`; +``` + +## Contributing Guidelines + +### Adding New File Formats +1. Implement `RemoteH5FileX` interface in `src/remote-h5-file/lib/` +2. Add format detection logic in `hdf5Interface.ts` +3. Update `getMergedRemoteH5File` for multi-file support + +### Performance Considerations +- Always prefer LINDI files when available +- Implement proper caching for new data types +- Use Web Workers for CPU-intensive operations +- Consider memory usage for large datasets + +### Testing Large Files +- Test with files >1GB to verify chunking works +- Verify LINDI fallback mechanisms +- Test authentication flows with embargoed data +- Check error handling for network failures + +### Plugin Development +- Use provided hooks (`useHdf5Group`, `useHdf5Dataset`, etc.) +- Implement proper loading states and error handling +- Consider data slicing for large arrays +- Follow lazy loading patterns + +## Debugging and Monitoring + +### Status Bar Integration +The system provides real-time statistics in the status bar: +- `numGroups / numDatasets / numDatasetDatas`: Operation counters +- Loading indicators for active operations +- Error notifications for failed requests + +### Console Logging +- LINDI detection attempts and results +- Authentication error details +- Performance metrics and timing +- Cache hit/miss information + +### Common Issues +1. **CORS Errors**: Usually resolved by LINDI files or proper headers +2. **Authentication Failures**: Check DANDI API key configuration +3. **Large Dataset Errors**: Implement proper slicing +4. **Worker Loading Failures**: Verify CDN accessibility + +## Future Improvements + +### Potential Optimizations +- Implement progressive loading for very large datasets +- Add compression support for data transfers +- Enhance caching with persistence across sessions +- Improve error recovery mechanisms + +### Format Extensions +- Support for additional HDF5-compatible formats +- Enhanced LINDI features (compression, encryption) +- Integration with cloud storage providers +- Real-time streaming capabilities + +This architecture enables Neurosift to efficiently handle NWB files ranging from megabytes to gigabytes while providing responsive user interactions and comprehensive error handling. diff --git a/package-lock.json b/package-lock.json index 7dbf2c5f..3371a380 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,6 @@ "": { "name": "neurosift", "version": "0.0.0", - "hasInstallScript": true, "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", @@ -18,6 +17,7 @@ "@types/plotly.js": "^2.35.2", "@types/react-syntax-highlighter": "^15.5.13", "@vercel/analytics": "^1.4.1", + "downsample": "^1.4.0", "mathjs": "^14.2.1", "nifti-reader-js": "^0.7.0", "numcodecs": "^0.3.1", @@ -3915,6 +3915,11 @@ "csstype": "^3.0.2" } }, + "node_modules/downsample": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/downsample/-/downsample-1.4.0.tgz", + "integrity": "sha512-teYPhUPxqwtyICt47t1mP/LjhbRV/ghuKb/LmFDbcZ0CjqFD31tn6rVLZoeCEa1xr8+f2skW8UjRiLiGIKQE4w==" + }, "node_modules/draw-svg-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/draw-svg-path/-/draw-svg-path-1.0.0.tgz", diff --git a/package.json b/package.json index 3e6e7901..036f744b 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@types/plotly.js": "^2.35.2", "@types/react-syntax-highlighter": "^15.5.13", "@vercel/analytics": "^1.4.1", + "downsample": "^1.4.0", "mathjs": "^14.2.1", "nifti-reader-js": "^0.7.0", "numcodecs": "^0.3.1", diff --git a/python/neurosift/TemporaryDirectory.py b/python/neurosift/TemporaryDirectory.py index 477e281c..a84575a0 100644 --- a/python/neurosift/TemporaryDirectory.py +++ b/python/neurosift/TemporaryDirectory.py @@ -4,40 +4,46 @@ import random import tempfile -class TemporaryDirectory(): - def __init__(self, *, remove: bool=True, prefix: str='tmp'): + +class TemporaryDirectory: + def __init__(self, *, remove: bool = True, prefix: str = "tmp"): self._remove = remove self._prefix = prefix def __enter__(self) -> str: tmpdir = tempfile.gettempdir() - self._path = f'{tmpdir}/{self._prefix}_{_random_string(8)}' + self._path = f"{tmpdir}/{self._prefix}_{_random_string(8)}" os.mkdir(self._path) return self._path def __exit__(self, exc_type, exc_val, exc_tb): if self._remove: - if not os.getenv('KACHERY_CLOUD_KEEP_TEMP_FILES') == '1': + if not os.getenv("KACHERY_CLOUD_KEEP_TEMP_FILES") == "1": _rmdir_with_retries(self._path, num_retries=5) def path(self): return self._path -def _rmdir_with_retries(dirname: str, num_retries: int, delay_between_tries: float=1): +def _rmdir_with_retries(dirname: str, num_retries: int, delay_between_tries: float = 1): for retry_num in range(1, num_retries + 1): if not os.path.exists(dirname): return try: shutil.rmtree(dirname) break - except: # pragma: no cover + except: # pragma: no cover if retry_num < num_retries: - print('Retrying to remove directory: {}'.format(dirname)) + print("Retrying to remove directory: {}".format(dirname)) time.sleep(delay_between_tries) else: - raise Exception('Unable to remove directory after {} tries: {}'.format(num_retries, dirname)) + raise Exception( + "Unable to remove directory after {} tries: {}".format( + num_retries, dirname + ) + ) + def _random_string(num_chars: int) -> str: - chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - return ''.join(random.choice(chars) for _ in range(num_chars)) \ No newline at end of file + chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + return "".join(random.choice(chars) for _ in range(num_chars)) diff --git a/python/neurosift/cli.py b/python/neurosift/cli.py index 23454973..c3fd2f1d 100644 --- a/python/neurosift/cli.py +++ b/python/neurosift/cli.py @@ -14,14 +14,18 @@ def neurosift(): @click.command() -@click.argument('file', type=click.Path(exists=True)) -@click.option('--neurosift-url', default='https://neurosift.app', help='Neurosift server URL (default: https://neurosift.app)') +@click.argument("file", type=click.Path(exists=True)) +@click.option( + "--neurosift-url", + default="https://neurosift.app", + help="Neurosift server URL (default: https://neurosift.app)", +) def view_nwb(file: str, neurosift_url: str): abs_fname = os.path.abspath(file) base_fname = os.path.basename(abs_fname) with TemporaryDirectory(prefix="view_nwb") as tmpdir: # create a symbolic link to the file (or zarr folder) - os.symlink(abs_fname, f'{tmpdir}/{base_fname}') + os.symlink(abs_fname, f"{tmpdir}/{base_fname}") # this directory this_directory = os.path.dirname(os.path.realpath(__file__)) @@ -29,55 +33,83 @@ def view_nwb(file: str, neurosift_url: str): env = os.environ.copy() # apparently shell=True is necessary for Windows, but shell=False is necessary for Linux - if os.name == 'nt': + if os.name == "nt": shell = True - elif os.name == 'posix': + elif os.name == "posix": shell = False else: - print(f'Warning: unrecognized os.name: {os.name}') + print(f"Warning: unrecognized os.name: {os.name}") shell = False try: - npm_version = subprocess.run(["npm", "--version"], stdout=subprocess.PIPE, universal_newlines=True, shell=shell, env=env).stdout.strip() - print(f'npm version: {npm_version}') + npm_version = subprocess.run( + ["npm", "--version"], + stdout=subprocess.PIPE, + universal_newlines=True, + shell=shell, + env=env, + ).stdout.strip() + print(f"npm version: {npm_version}") except Exception: - raise Exception('Unable to run npm.') + raise Exception("Unable to run npm.") try: - node_version = subprocess.run(["node", "--version"], stdout=subprocess.PIPE, universal_newlines=True, shell=shell, env=env).stdout.strip() - print(f'node version: {node_version}') + node_version = subprocess.run( + ["node", "--version"], + stdout=subprocess.PIPE, + universal_newlines=True, + shell=shell, + env=env, + ).stdout.strip() + print(f"node version: {node_version}") except Exception: - raise Exception('Unable to run node.') + raise Exception("Unable to run node.") # parse node_version v18.0.0 to get the major version number - node_major_version = int(node_version.split('.')[0][1:]) + node_major_version = int(node_version.split(".")[0][1:]) if node_major_version < 16: - raise Exception('node version must be >= 16.0.0') + raise Exception("node version must be >= 16.0.0") # run the command npm install in the js directory - subprocess.run(["npm", "install"], cwd=f'{this_directory}/local-file-access-js', shell=shell, env=env) + subprocess.run( + ["npm", "install"], + cwd=f"{this_directory}/local-file-access-js", + shell=shell, + env=env, + ) # find an open port port = find_free_port() # run the service - env['PORT'] = str(port) - process = subprocess.Popen(['npm', 'run', 'start', tmpdir], cwd=f'{this_directory}/local-file-access-js', shell=shell, env=env) - - zarr_param = '' + env["PORT"] = str(port) + process = subprocess.Popen( + ["npm", "run", "start", tmpdir], + cwd=f"{this_directory}/local-file-access-js", + shell=shell, + env=env, + ) + + zarr_param = "" if os.path.isdir(abs_fname): - if not os.path.exists(f'{abs_fname}/.zmetadata'): - raise Exception(f'{abs_fname} is a directory but does not contain a .zmetadata file.') - zarr_param = '&zarr=1' + if not os.path.exists(f"{abs_fname}/.zmetadata"): + raise Exception( + f"{abs_fname} is a directory but does not contain a .zmetadata file." + ) + zarr_param = "&zarr=1" # it's important to wait a bit before opening the browser time.sleep(3) # open the browser url = f"{neurosift_url}/?p=/nwb&url=http://localhost:{port}/files/{base_fname}{zarr_param}" - if file.endswith('.lindi') or file.endswith('.lindi.tar') or file.endswith('.lindi.json'): - url = url + '&st=lindi' - print(f'Opening {url}') + if ( + file.endswith(".lindi") + or file.endswith(".lindi.tar") + or file.endswith(".lindi.json") + ): + url = url + "&st=lindi" + print(f"Opening {url}") webbrowser.open(url) # wait for the process to finish @@ -86,7 +118,7 @@ def view_nwb(file: str, neurosift_url: str): def find_free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.bind(('', 0)) + s.bind(("", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) return s.getsockname()[1] @@ -95,5 +127,5 @@ def find_free_port(): neurosift.add_command(view_nwb) -if __name__ == '__main__': +if __name__ == "__main__": neurosift() diff --git a/python/neurosift/codecs/MP4AVCCodec.py b/python/neurosift/codecs/MP4AVCCodec.py index cab53ce2..758d32ef 100644 --- a/python/neurosift/codecs/MP4AVCCodec.py +++ b/python/neurosift/codecs/MP4AVCCodec.py @@ -12,12 +12,10 @@ class MP4AVCCodec(Codec): -------- See examples/example_mp4avc_codec.py in the neurosift repo. """ + codec_id = "mp4avc" - def __init__( - self, - fps: float - ): + def __init__(self, fps: float): """ Parameters ---------- @@ -50,13 +48,19 @@ def encode(self, array: np.ndarray): # type: ignore is_color = True with tempfile.TemporaryDirectory() as tmpdir: - tmp_output_fname = f'{tmpdir}/output.mp4' - fourcc = cv2.VideoWriter_fourcc(*'avc1') # type: ignore - writer = cv2.VideoWriter(tmp_output_fname, fourcc, self.fps, (array.shape[2], array.shape[1]), isColor=is_color) + tmp_output_fname = f"{tmpdir}/output.mp4" + fourcc = cv2.VideoWriter_fourcc(*"avc1") # type: ignore + writer = cv2.VideoWriter( + tmp_output_fname, + fourcc, + self.fps, + (array.shape[2], array.shape[1]), + isColor=is_color, + ) for i in range(array.shape[0]): writer.write(array[i]) writer.release() - with open(tmp_output_fname, 'rb') as f: + with open(tmp_output_fname, "rb") as f: return f.read() def decode(self, buf: bytes, out=None): # type: ignore @@ -73,9 +77,10 @@ def decode(self, buf: bytes, out=None): # type: ignore (frames, height, width, 3) or (frames, height, width). """ import cv2 + with tempfile.TemporaryDirectory() as tmpdir: - tmp_input_fname = f'{tmpdir}/input.mp4' - with open(tmp_input_fname, 'wb') as f: + tmp_input_fname = f"{tmpdir}/input.mp4" + with open(tmp_input_fname, "wb") as f: f.write(buf) cap = cv2.VideoCapture(tmp_input_fname) frames = [] @@ -91,9 +96,7 @@ def decode(self, buf: bytes, out=None): # type: ignore return ret def __repr__(self): - return ( - f'{self.__class__.__name__}(fps={self.fps})' - ) + return f"{self.__class__.__name__}(fps={self.fps})" @staticmethod def register_codec(): diff --git a/python/neurosift/local-file-access-js/src/index.js b/python/neurosift/local-file-access-js/src/index.js index a26cbe30..b1ec9ff1 100644 --- a/python/neurosift/local-file-access-js/src/index.js +++ b/python/neurosift/local-file-access-js/src/index.js @@ -10,7 +10,13 @@ if (!dir) { console.info('Serving files in', dir) // Allow CORS from neurosift.app flatironinstitute.github.io and localhost:3000 -const allowedOrigins = ['https://neurosift.app', 'https://flatironinstitute.github.io', 'http://localhost:3000', 'http://localhost:4200'] +const allowedOrigins = [ + 'https://neurosift.app', + 'https://flatironinstitute.github.io', + 'http://localhost:3000', + 'http://localhost:4200', + 'http://localhost:5173' // local dev server for neurosift +] app.use((req, resp, next) => { const origin = req.get('origin') const allowedOrigin = allowedOrigins.includes(origin) ? origin : undefined diff --git a/src/pages/NwbPage/plugins/IntracellularRecordingsTable/IntracellularRecordingsTableView.tsx b/src/pages/NwbPage/plugins/IntracellularRecordingsTable/IntracellularRecordingsTableView.tsx new file mode 100644 index 00000000..5bfba41d --- /dev/null +++ b/src/pages/NwbPage/plugins/IntracellularRecordingsTable/IntracellularRecordingsTableView.tsx @@ -0,0 +1,129 @@ +import React from "react"; +import { useHdf5Group } from "@hdf5Interface"; + +type Props = { + nwbUrl: string; + path: string; + width?: number; + height?: number; + objectType: "group" | "dataset"; + onOpenObjectInNewTab?: (path: string) => void; +}; + +const IntracellularRecordingsTableView: React.FC = ({ + nwbUrl, + path, + width = 500, + height = 400, +}) => { + const group = useHdf5Group(nwbUrl, path); + + if (!group) { + return
Loading IntracellularRecordingsTable...
; + } + + return ( +
+

+ IntracellularRecordingsTable +

+ +
+ Path: {path} +
+ +
+ Attributes: +
+ +
+ {Object.keys(group.attrs).length === 0 ? ( +
+ No attributes found +
+ ) : ( + Object.entries(group.attrs).map(([key, value]) => ( +
+
{key}:
+
+ {typeof value === "object" + ? JSON.stringify(value, null, 2) + : String(value)} +
+
+ )) + )} +
+ + {/* Show basic structure info */} +
+ Structure: +
+
+
Subgroups: {group.subgroups.length}
+
Datasets: {group.datasets.length}
+ + {group.subgroups.length > 0 && ( +
+
Subgroups:
+ {group.subgroups.map((sg) => ( +
+ • {sg.name} +
+ ))} +
+ )} + + {group.datasets.length > 0 && ( +
+
Datasets:
+ {group.datasets.map((ds) => ( +
+ • {ds.name} ({ds.dtype}, shape: {JSON.stringify(ds.shape)}) +
+ ))} +
+ )} +
+
+ ); +}; + +export default IntracellularRecordingsTableView; diff --git a/src/pages/NwbPage/plugins/IntracellularRecordingsTable/index.ts b/src/pages/NwbPage/plugins/IntracellularRecordingsTable/index.ts new file mode 100644 index 00000000..1c0b2c9d --- /dev/null +++ b/src/pages/NwbPage/plugins/IntracellularRecordingsTable/index.ts @@ -0,0 +1,21 @@ +import { getHdf5Group } from "@hdf5Interface"; +import { NwbObjectViewPlugin } from "../pluginInterface"; +import IntracellularRecordingsTableView from "./IntracellularRecordingsTableView"; + +export const intracellularRecordingsTablePlugin: NwbObjectViewPlugin = { + name: "Icephys", + canHandle: async ({ nwbUrl, path }: { nwbUrl: string; path: string }) => { + const group = await getHdf5Group(nwbUrl, path); + if (!group) return false; + + // Check if this is an IntracellularRecordingsTable neurodata_type + if (group.attrs.neurodata_type !== "IntracellularRecordingsTable") { + return false; + } + + return true; + }, + component: IntracellularRecordingsTableView, + showInMultiView: true, + launchableFromTable: true, +}; diff --git a/src/pages/NwbPage/plugins/SequentialRecordingsTable/SequentialRecordingsPlotly.tsx b/src/pages/NwbPage/plugins/SequentialRecordingsTable/SequentialRecordingsPlotly.tsx new file mode 100644 index 00000000..c0046605 --- /dev/null +++ b/src/pages/NwbPage/plugins/SequentialRecordingsTable/SequentialRecordingsPlotly.tsx @@ -0,0 +1,515 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import React, { useMemo, useState } from "react"; +import Plot from "react-plotly.js"; +import { Data, Layout } from "plotly.js"; +import { useSequentialRecordingsData } from "./useSequentialRecordingsData"; + +const colors = [ + "#1f77b4", + "#ff7f0e", + "#2ca02c", + "#d62728", + "#9467bd", + "#8c564b", + "#e377c2", + "#7f7f7f", + "#bcbd22", + "#17becf", + "#aec7e8", + "#ffbb78", + "#98df8a", + "#ff9896", + "#c5b0d5", + "#c49c94", + "#f7b6d3", + "#c7c7c7", + "#dbdb8d", + "#9edae5", +]; + +type Props = { + nwbUrl: string; + path: string; + width?: number; + height?: number; + timeRange?: { start: number; duration: number }; +}; + +const SequentialRecordingsPlotly: React.FC = ({ + nwbUrl, + path, + width = 1100, + height = 700, + timeRange, +}) => { + const [selectedStimulusType, setSelectedStimulusType] = useState(""); + const [visiblePairs, setVisiblePairs] = useState>(new Set()); + + // UI state for controls (pending changes) + const [downsampleMethod, setDownsampleMethod] = useState("lttb"); + const [downsampleFactor, setDownsampleFactor] = useState(10); + + // Applied state for actual data loading + const [appliedDownsampleMethod, setAppliedDownsampleMethod] = + useState("lttb"); + const [appliedDownsampleFactor, setAppliedDownsampleFactor] = + useState(10); + + const appliedDownsampleOptions = useMemo( + () => ({ + downsampleMethod: appliedDownsampleMethod, + downsampleFactor: appliedDownsampleFactor, + }), + [appliedDownsampleMethod, appliedDownsampleFactor], + ); + + const { pairs, stimulusTypes, isLoading, error } = + useSequentialRecordingsData( + nwbUrl, + path, + timeRange, + appliedDownsampleOptions, + ); + + // Check if settings have changed + const hasUnappliedChanges = + downsampleMethod !== appliedDownsampleMethod || + downsampleFactor !== appliedDownsampleFactor; + + // Handle downsample button click + const handleDownsample = () => { + setAppliedDownsampleMethod(downsampleMethod); + setAppliedDownsampleFactor(downsampleFactor); + }; + + // Initialize visible pairs and stimulus type when data loads + React.useEffect(() => { + if (pairs.length > 0 && visiblePairs.size === 0) { + setVisiblePairs(new Set(pairs.map((p) => p.pairId))); + } + if (stimulusTypes.length > 0 && selectedStimulusType === "") { + setSelectedStimulusType(stimulusTypes[0]); + } + }, [pairs, visiblePairs.size, stimulusTypes, selectedStimulusType]); + + // Filter pairs based on selected stimulus type + const filteredPairs = useMemo(() => { + return pairs.filter((pair) => pair.stimulusType === selectedStimulusType); + }, [pairs, selectedStimulusType]); + + // Create Plotly data for overlapping traces with relative times + const plotData = useMemo(() => { + const traces: Data[] = []; + + filteredPairs.forEach((pair) => { + const color = colors[pair.pairId % colors.length]; + const label = `${pair.stimulusType} #${pair.pairId}`; + const isVisible = visiblePairs.has(pair.pairId); + + // Normalize timestamps to start at zero + const stimulusStartTime = pair.stimulusData.timestamps[0] || 0; + const responseStartTime = pair.responseData.timestamps[0] || 0; + + const relativeStimTimes = pair.stimulusData.timestamps.map( + (t) => t - stimulusStartTime, + ); + const relativeRespTimes = pair.responseData.timestamps.map( + (t) => t - responseStartTime, + ); + + // Stimulus trace (left subplot) + traces.push({ + x: relativeStimTimes, + y: pair.stimulusData.data, + type: "scatter", + mode: "lines", + line: { color, width: 2 }, + name: `${label}`, + legendgroup: `pair-${pair.pairId}`, + showlegend: true, + visible: isVisible ? true : "legendonly", + xaxis: "x", + yaxis: "y", + hovertemplate: + `${label} (Stimulus)
` + + "Time: %{x:.3f}s
" + + "Value: %{y:.3f}
" + + "", + }); + + // Response trace (right subplot) + traces.push({ + x: relativeRespTimes, + y: pair.responseData.data, + type: "scatter", + mode: "lines", + line: { color, width: 2 }, + name: `${label}`, + legendgroup: `pair-${pair.pairId}`, + showlegend: false, + visible: isVisible ? true : "legendonly", + xaxis: "x2", + yaxis: "y2", + hovertemplate: + `${label} (Response)
` + + "Time: %{x:.3f}s
" + + "Value: %{y:.3f}
" + + "", + }); + }); + + return traces; + }, [filteredPairs, visiblePairs]); + + // Extract units from the first available pair for axis labeling + const stimulusUnit = + filteredPairs.length > 0 ? filteredPairs[0].stimulusData.unit : "unit"; + const responseUnit = + filteredPairs.length > 0 ? filteredPairs[0].responseData.unit : "unit"; + const timeUnit = + filteredPairs.length > 0 + ? filteredPairs[0].stimulusData.timeUnit || "seconds" + : "seconds"; + + // Layout configuration for dual subplots using proper subplot approach + const layout: Partial = useMemo( + () => ({ + title: { + text: `Sequential Recordings - ${selectedStimulusType}`, + x: 0.5, + xanchor: "center", + }, + width: width - 20, + height: height - 20, + margin: { l: 80, r: 80, t: 80, b: 150 }, + + // Configure subplots with proper grid + grid: { + rows: 1, + columns: 2, + pattern: "independent", + xgap: 0.1, + }, + + // Left subplot (Stimulus) - subplot 1 + xaxis: { + title: { + text: `Time [${timeUnit}]`, + font: { size: 14, color: "#333" }, + }, + domain: [0, 0.45], + showgrid: true, + gridcolor: "#e0e0e0", + }, + yaxis: { + title: { + text: `Stimulus [${stimulusUnit}]`, + font: { size: 14, color: "#333" }, + }, + showgrid: true, + gridcolor: "#e0e0e0", + }, + + // Right subplot (Response) - subplot 2 + xaxis2: { + title: { + text: `Time [${timeUnit}]`, + font: { size: 14, color: "#333" }, + }, + domain: [0.55, 1], + showgrid: true, + gridcolor: "#e0e0e0", + }, + yaxis2: { + title: { + text: `Response [${responseUnit}]`, + font: { size: 14, color: "#333" }, + }, + side: "left", + showgrid: true, + gridcolor: "#e0e0e0", + }, + + // Annotations for subplot titles + annotations: [ + { + text: "Stimulus", + x: 0.225, + y: 1.02, + xref: "paper", + yref: "paper", + xanchor: "center", + yanchor: "bottom", + showarrow: false, + font: { size: 16, color: "#333" }, + }, + { + text: "Response", + x: 0.775, + y: 1.02, + xref: "paper", + yref: "paper", + xanchor: "center", + yanchor: "bottom", + showarrow: false, + font: { size: 16, color: "#333" }, + }, + ], + + legend: { + title: { text: "Pairs" }, + x: 1.02, + y: 1, + xanchor: "left", + yanchor: "top", + bgcolor: "rgba(255,255,255,0.8)", + bordercolor: "#ccc", + borderwidth: 1, + }, + + plot_bgcolor: "white", + paper_bgcolor: "white", + dragmode: "zoom", + }), + [width, height, selectedStimulusType], + ); + + const config = useMemo( + () => ({ + responsive: true, + displayModeBar: true, + modeBarButtonsToRemove: ["lasso2d", "select2d"] as any, + displaylogo: false, + }), + [], + ); + + // Handle legend click to toggle pair visibility + const handleLegendClick = (event: any) => { + if (event && event.curveNumber !== undefined) { + // Get the trace that was clicked + const trace = plotData[event.curveNumber]; + if (trace && (trace as any).legendgroup) { + const legendgroup = (trace as any).legendgroup; + const pairId = parseInt(legendgroup.split("-")[1]); + + setVisiblePairs((prev) => { + const newSet = new Set(prev); + if (newSet.has(pairId)) { + newSet.delete(pairId); + } else { + newSet.add(pairId); + } + return newSet; + }); + + // Return false to prevent Plotly's default legend behavior + return false; + } + } + + // Allow default behavior for other cases + return true; + }; + + if (isLoading) { + return ( +
+ Loading sequential recordings data... +
+ ); + } + + if (error) { + return ( +
+
+ Error loading data: +
+ {error} +
+
+ ); + } + + if (pairs.length === 0) { + return ( +
+ No sequential recordings found in this dataset. +
+ ); + } + + return ( +
+ {/* Controls */} +
+
+ + + + + + + +
+ + {/* Downsampling warning message inside controls */} + {appliedDownsampleFactor > 1 && ( +
+ ⚠️ The traces below were downsampled for performance, the raw traces + may be different. +
+ )} +
+ + {/* Plot */} + +
+ ); +}; + +export default SequentialRecordingsPlotly; diff --git a/src/pages/NwbPage/plugins/SequentialRecordingsTable/SequentialRecordingsTableView.tsx b/src/pages/NwbPage/plugins/SequentialRecordingsTable/SequentialRecordingsTableView.tsx new file mode 100644 index 00000000..51619095 --- /dev/null +++ b/src/pages/NwbPage/plugins/SequentialRecordingsTable/SequentialRecordingsTableView.tsx @@ -0,0 +1,105 @@ +import React from "react"; +import { useHdf5Group } from "@hdf5Interface"; +import SequentialRecordingsPlotly from "./SequentialRecordingsPlotly"; +import { useTimeseriesSelection } from "@shared/context-timeseries-selection-2"; + +type Props = { + nwbUrl: string; + path: string; + width?: number; + height?: number; + objectType: "group" | "dataset"; + onOpenObjectInNewTab?: (path: string) => void; +}; + +const SequentialRecordingsTableView: React.FC = ({ + nwbUrl, + path, + width = 1200, + height = 800, +}) => { + const group = useHdf5Group(nwbUrl, path); + const { visibleStartTimeSec, visibleEndTimeSec } = useTimeseriesSelection(); + + if (!group) { + return
Loading SequentialRecordingsTable...
; + } + + // Create time range object for the visualization + const timeRange = + visibleStartTimeSec !== undefined && visibleEndTimeSec !== undefined + ? { + start: visibleStartTimeSec, + duration: visibleEndTimeSec - visibleStartTimeSec, + } + : undefined; + + return ( +
+

+ Sequential Recordings Table +

+ +
+ Path: {path} +
+ + {/* Description/Instructions */} +
+
+ 📊 Sequential Recordings Visualization +
+
+ This table contains intracellular electrophysiology recordings + organized by stimulus type. +
+ {Object.keys(group.attrs).length > 0 && ( +
+ Description:{" "} + {group.attrs.description || "No description available"} +
+ )} +
+ + {/* Visualization */} +
+ +
+
+ ); +}; + +export default SequentialRecordingsTableView; diff --git a/src/pages/NwbPage/plugins/SequentialRecordingsTable/index.ts b/src/pages/NwbPage/plugins/SequentialRecordingsTable/index.ts new file mode 100644 index 00000000..6f8f4f17 --- /dev/null +++ b/src/pages/NwbPage/plugins/SequentialRecordingsTable/index.ts @@ -0,0 +1,21 @@ +import { getHdf5Group } from "@hdf5Interface"; +import { NwbObjectViewPlugin } from "../pluginInterface"; +import SequentialRecordingsTableView from "./SequentialRecordingsTableView"; + +export const sequentialRecordingsTablePlugin: NwbObjectViewPlugin = { + name: "Sequential", + canHandle: async ({ nwbUrl, path }: { nwbUrl: string; path: string }) => { + const group = await getHdf5Group(nwbUrl, path); + if (!group) return false; + + // Check if this is a SequentialRecordingsTable neurodata_type + if (group.attrs.neurodata_type !== "SequentialRecordingsTable") { + return false; + } + + return true; + }, + component: SequentialRecordingsTableView, + showInMultiView: true, + launchableFromTable: true, +}; diff --git a/src/pages/NwbPage/plugins/SequentialRecordingsTable/types.ts b/src/pages/NwbPage/plugins/SequentialRecordingsTable/types.ts new file mode 100644 index 00000000..418f00d2 --- /dev/null +++ b/src/pages/NwbPage/plugins/SequentialRecordingsTable/types.ts @@ -0,0 +1,27 @@ +export interface TimeseriesDataWithUnits { + timestamps: number[]; + data: number[]; + unit: string; + timeUnit?: string; +} + +export interface SequentialRecordingsPair { + pairId: number; + stimulusType: string; + stimulusPath: string; + responsePath: string; + stimulusData: TimeseriesDataWithUnits; + responseData: TimeseriesDataWithUnits; +} + +export interface SequentialRecordingsData { + pairs: SequentialRecordingsPair[]; + stimulusTypes: string[]; + isLoading: boolean; + error?: string; +} + +export interface TimeRange { + start: number; + duration: number; +} diff --git a/src/pages/NwbPage/plugins/SequentialRecordingsTable/useSequentialRecordingsData.ts b/src/pages/NwbPage/plugins/SequentialRecordingsTable/useSequentialRecordingsData.ts new file mode 100644 index 00000000..e5e218f8 --- /dev/null +++ b/src/pages/NwbPage/plugins/SequentialRecordingsTable/useSequentialRecordingsData.ts @@ -0,0 +1,330 @@ +import { getHdf5DatasetData, getHdf5Group } from "@hdf5Interface"; +import { useEffect, useState } from "react"; +import { ChunkedTimeseriesClient } from "../simple-timeseries/TimeseriesClient"; +import { DownsampledChunkedTimeseriesClient } from "../simple-timeseries/DownsampledTimeseriesClient"; +import { + SequentialRecordingsData, + SequentialRecordingsPair, + TimeRange, + TimeseriesDataWithUnits, +} from "./types"; + +export const useSequentialRecordingsData = ( + nwbUrl: string, + path: string, + timeRange?: TimeRange, + downsampleOptions?: { downsampleMethod: string; downsampleFactor: number }, +): SequentialRecordingsData => { + const [data, setData] = useState({ + pairs: [], + stimulusTypes: [], + isLoading: true, + error: undefined, + }); + + useEffect(() => { + let cancelled = false; + + const loadData = async () => { + try { + setData((prev) => ({ ...prev, isLoading: true, error: undefined })); + + // Step 1: Load stimulus types from sequential recordings + const stimulusTypesData = await getHdf5DatasetData( + nwbUrl, + `${path}/stimulus_type`, + {}, + ); + if (!stimulusTypesData) { + throw new Error("Could not load stimulus types"); + } + + // Step 2: Load simultaneous recordings indices + const simultaneousRecordingsData = await getHdf5DatasetData( + nwbUrl, + `${path}/simultaneous_recordings`, + {}, + ); + const simultaneousRecordingsIndex = await getHdf5DatasetData( + nwbUrl, + `${path}/simultaneous_recordings_index`, + {}, + ); + + if (!simultaneousRecordingsData || !simultaneousRecordingsIndex) { + throw new Error("Could not load simultaneous recordings data"); + } + + // Step 3: Load recordings from simultaneous_recordings table + const recordingsData = await getHdf5DatasetData( + nwbUrl, + "/general/intracellular_ephys/simultaneous_recordings/recordings", + {}, + ); + const recordingsIndex = await getHdf5DatasetData( + nwbUrl, + "/general/intracellular_ephys/simultaneous_recordings/recordings_index", + {}, + ); + + if (!recordingsData || !recordingsIndex) { + throw new Error("Could not load recordings data"); + } + + // Step 4: Load stimulus and response references + const stimulusRefs = await getHdf5DatasetData( + nwbUrl, + "/general/intracellular_ephys/intracellular_recordings/stimuli/stimulus", + {}, + ); + const responseRefs = await getHdf5DatasetData( + nwbUrl, + "/general/intracellular_ephys/intracellular_recordings/responses/response", + {}, + ); + + if (!stimulusRefs || !responseRefs) { + throw new Error("Could not load stimulus/response references"); + } + + if (cancelled) return; + + // Step 5: Process the data structure + const pairs: SequentialRecordingsPair[] = []; + const stimulusTypesSet = new Set(); + let pairId = 0; + + // Convert data to arrays if needed + const stimulusTypesArray = Array.isArray(stimulusTypesData) + ? stimulusTypesData + : Array.from(stimulusTypesData as any); + const simultaneousRecordingsArray = Array.isArray( + simultaneousRecordingsData, + ) + ? simultaneousRecordingsData + : Array.from(simultaneousRecordingsData as any); + const simultaneousRecordingsIndexArray = Array.isArray( + simultaneousRecordingsIndex, + ) + ? simultaneousRecordingsIndex + : Array.from(simultaneousRecordingsIndex as any); + const recordingsArray = Array.isArray(recordingsData) + ? recordingsData + : Array.from(recordingsData as any); + const recordingsIndexArray = Array.isArray(recordingsIndex) + ? recordingsIndex + : Array.from(recordingsIndex as any); + + // Helper function to get ragged array slice + const getRaggedSlice = (array: any[], index: any[], row: number) => { + const start = row === 0 ? 0 : index[row - 1]; + const end = index[row]; + return array.slice(start, end); + }; + + // Iterate through sequential recordings + for (let seqRow = 0; seqRow < stimulusTypesArray.length; seqRow++) { + const stimulusType = String(stimulusTypesArray[seqRow]); + stimulusTypesSet.add(stimulusType); + + // Get simultaneous recordings for this sequential recording + const simRecordings = getRaggedSlice( + simultaneousRecordingsArray, + simultaneousRecordingsIndexArray, + seqRow, + ); + + // For each simultaneous recording + for (const simRow of simRecordings) { + // Get individual recordings for this simultaneous recording + const recordings = getRaggedSlice( + recordingsArray, + recordingsIndexArray, + simRow, + ); + + // For each individual recording + for (const recId of recordings) { + if (cancelled) return; + + try { + // Get stimulus and response references + const stimRef = (stimulusRefs as any)[recId]; + const respRef = (responseRefs as any)[recId]; + + if (!stimRef || !respRef) { + continue; + } + + // Extract timeseries paths from compound dataset references + // The refs are arrays with [idx_start, count, timeseries_object_ref] + let stimulusPath: string; + let responsePath: string; + + if (Array.isArray(stimRef) && stimRef.length >= 3) { + const stimTimeseriesRef = stimRef[2]; + + // Try to decode the Uint8Array object reference + if (stimTimeseriesRef instanceof Uint8Array) { + // Construct the stimulus path based on the recording ID + // E.g.: Recording ID 3 maps to stimulus-04-ch-0 (ID + 1) + stimulusPath = `/stimulus/presentation/stimulus-${String(recId + 1).padStart(2, "0")}-ch-0`; + } else { + stimulusPath = String(stimTimeseriesRef); + } + } else { + console.warn( + `Unexpected stimulus ref structure for ${recId}:`, + stimRef, + ); + continue; + } + + if (Array.isArray(respRef) && respRef.length >= 3) { + const respTimeseriesRef = respRef[2]; + + // Try to decode the Uint8Array object reference + if (respTimeseriesRef instanceof Uint8Array) { + // Construct the response path based on the recording ID + // E.g.: Recording ID 3 maps to current_clamp-response-04-ch-0 (ID + 1) + responsePath = `/acquisition/current_clamp-response-${String(recId + 1).padStart(2, "0")}-ch-0`; + } else { + responsePath = String(respTimeseriesRef); + } + } else { + console.warn( + `Unexpected response ref structure for ${recId}:`, + respRef, + ); + continue; + } + + // Load timeseries data for stimulus and response + const [stimulusData, responseData] = await Promise.all([ + loadTimeseriesData( + nwbUrl, + stimulusPath, + timeRange, + downsampleOptions, + ), + loadTimeseriesData( + nwbUrl, + responsePath, + timeRange, + downsampleOptions, + ), + ]); + + if (stimulusData && responseData) { + pairs.push({ + pairId, + stimulusType, + stimulusPath, + responsePath, + stimulusData, + responseData, + }); + pairId++; + } + } catch (error) { + console.warn(`Failed to load pair ${pairId}:`, error); + // Continue with next recording + } + } + } + } + + if (cancelled) return; + + setData({ + pairs, + stimulusTypes: Array.from(stimulusTypesSet).sort(), + isLoading: false, + error: undefined, + }); + } catch (error) { + if (cancelled) return; + console.error("Error loading sequential recordings data:", error); + setData((prev) => ({ + ...prev, + isLoading: false, + error: error instanceof Error ? error.message : "Unknown error", + })); + } + }; + + loadData(); + + return () => { + cancelled = true; + }; + }, [ + nwbUrl, + path, + timeRange, + downsampleOptions?.downsampleMethod, + downsampleOptions?.downsampleFactor, + ]); + + return data; +}; + +// Helper function to load timeseries data from a path +const loadTimeseriesData = async ( + nwbUrl: string, + timeseriesPath: string, + timeRange?: TimeRange, + downsampleOptions?: { downsampleMethod: string; downsampleFactor: number }, +): Promise => { + try { + // Get the group for this timeseries + const group = await getHdf5Group(nwbUrl, timeseriesPath); + if (!group) return null; + + // Extract unit information from the data dataset attributes + const dataDataset = group.datasets.find((ds) => ds.name === "data"); + const unit = dataDataset?.attrs?.unit || "unknown"; + + // Extract time unit from timestamps dataset if available + const timestampsDataset = group.datasets.find( + (ds) => ds.name === "timestamps", + ); + const timeUnit = timestampsDataset?.attrs?.unit || "seconds"; + + // Create a timeseries client with optional downsampling + const downsampleFactor = downsampleOptions?.downsampleFactor ?? 10; + const downsampleMethod = downsampleOptions?.downsampleMethod ?? "lttb"; + const client = + downsampleFactor > 1 + ? await DownsampledChunkedTimeseriesClient.create(nwbUrl, group, { + downsampleFactor, + downsampleMethod: downsampleMethod as "decimate" | "lttb", + chunkSizeSec: 1, + }) + : await ChunkedTimeseriesClient.create(nwbUrl, group, { + chunkSizeSec: 1, + }); + + // Determine time range to load + const startTime = timeRange?.start ?? client.startTime; + const endTime = timeRange + ? timeRange.start + timeRange.duration + : Math.min(client.startTime + 10, client.endTime); // Default to first 10 seconds + + // Load data for first channel only (most intracellular recordings are single channel) + const result = await client.getDataForTimeRange(startTime, endTime, 0, 1); + + return { + timestamps: result.timestamps, + data: result.data[0] || [], // First channel + unit: String(unit), + timeUnit: String(timeUnit), + }; + } catch (error) { + console.warn( + `Failed to load timeseries data from ${timeseriesPath}:`, + error, + ); + return null; + } +}; diff --git a/src/pages/NwbPage/plugins/registry.ts b/src/pages/NwbPage/plugins/registry.ts index e7801fc3..186b5c15 100644 --- a/src/pages/NwbPage/plugins/registry.ts +++ b/src/pages/NwbPage/plugins/registry.ts @@ -19,6 +19,8 @@ import spikeDensityPlugin from "./SpikeDensity"; import { intervalSeriesPlugin } from "./IntervalSeries"; import { eventsPlugin } from "./Events"; import { imageSeriesMp4Plugin } from "./ImageSeriesMp4"; +import { intracellularRecordingsTablePlugin } from "./IntracellularRecordingsTable"; +import { sequentialRecordingsTablePlugin } from "./SequentialRecordingsTable"; import { NwbFileSpecifications } from "../SpecificationsView/SetupNwbFileSpecificationsProvider"; // List of plugins in order they will appear in the UI when a single object is being viewed @@ -28,6 +30,8 @@ export const nwbObjectViewPlugins: NwbObjectViewPlugin[] = [ behavioralEventsPlugin, dynamicTablePlugin, + intracellularRecordingsTablePlugin, + sequentialRecordingsTablePlugin, twoPhotonSeriesPlugin, spatialSeriesPlugin, simpleTimeseriesPlugin, diff --git a/src/pages/NwbPage/plugins/simple-timeseries/DownsampledTimeseriesClient.ts b/src/pages/NwbPage/plugins/simple-timeseries/DownsampledTimeseriesClient.ts new file mode 100644 index 00000000..34a55410 --- /dev/null +++ b/src/pages/NwbPage/plugins/simple-timeseries/DownsampledTimeseriesClient.ts @@ -0,0 +1,189 @@ +import { Hdf5Group } from "@hdf5Interface"; +import { ChunkedTimeseriesClient } from "./TimeseriesClient"; +import { LTTB } from "downsample"; + +export class DownsampledChunkedTimeseriesClient { + private baseClient: ChunkedTimeseriesClient; + private downsampleFactor: number; + private downsampleMethod: "decimate" | "lttb"; + + private constructor( + baseClient: ChunkedTimeseriesClient, + downsampleFactor: number, + downsampleMethod: "decimate" | "lttb" = "lttb", + ) { + this.baseClient = baseClient; + this.downsampleFactor = downsampleFactor; + this.downsampleMethod = downsampleMethod; + } + + static async create( + nwbUrl: string, + group: Hdf5Group, + options: { + downsampleFactor: number; + downsampleMethod?: "decimate" | "lttb"; + chunkSizeSec?: number; + chunkSizeNumChannels?: number; + }, + ): Promise { + const { + downsampleFactor, + downsampleMethod = "lttb", + ...chunkOptions + } = options; + const baseClient = await ChunkedTimeseriesClient.create( + nwbUrl, + group, + chunkOptions, + ); + return new DownsampledChunkedTimeseriesClient( + baseClient, + downsampleFactor, + downsampleMethod, + ); + } + + private downsampleDecimate( + timestamps: number[], + data: number[][], + factor: number, + ): { timestamps: number[]; data: number[][] } { + if (factor <= 1) { + return { timestamps, data }; + } + + const decimatedTimestamps = timestamps.filter((_, i) => i % factor === 0); + const decimatedData = data.map((channel) => + channel.filter((_, i) => i % factor === 0), + ); + + return { + timestamps: decimatedTimestamps, + data: decimatedData, + }; + } + + private downsampleLTTB( + timestamps: number[], + data: number[][], + factor: number, + ): { timestamps: number[]; data: number[][] } { + if (factor <= 1) { + return { timestamps, data }; + } + + const targetPoints = Math.ceil(timestamps.length / factor); + if (targetPoints >= timestamps.length) { + return { timestamps, data }; + } + + const downsampledData: number[][] = []; + let downsampledTimestamps: number[] = []; + + // Process each channel independently + for (let channelIndex = 0; channelIndex < data.length; channelIndex++) { + const channelData = data[channelIndex]; + + // Convert to [x, y] format required by LTTB + const xyData: [number, number][] = timestamps.map((t, i) => [ + t, + channelData[i], + ]); + + // Apply LTTB downsampling + const downsampled = LTTB(xyData, targetPoints) as [number, number][]; + + // Extract data for this channel + downsampledData.push( + downsampled.map((point: [number, number]) => point[1]), + ); + + // Use timestamps from first channel (all channels should have same timestamps) + if (channelIndex === 0) { + downsampledTimestamps = downsampled.map( + (point: [number, number]) => point[0], + ); + } + } + + return { + timestamps: downsampledTimestamps, + data: downsampledData, + }; + } + + async getDataForTimeRange( + tStart: number, + tEnd: number, + channelStart: number, + channelEnd: number, + ) { + // Get data from base client + const originalData = await this.baseClient.getDataForTimeRange( + tStart, + tEnd, + channelStart, + channelEnd, + ); + + // Apply selected downsampling method + return this.applyDownsampling( + originalData.timestamps, + originalData.data, + this.downsampleMethod, + this.downsampleFactor, + ); + } + + private applyDownsampling( + timestamps: number[], + data: number[][], + method: "decimate" | "lttb", + factor: number, + ): { timestamps: number[]; data: number[][] } { + switch (method) { + case "lttb": + return this.downsampleLTTB(timestamps, data, factor); + case "decimate": + default: + return this.downsampleDecimate(timestamps, data, factor); + } + } + + get startTime(): number { + return this.baseClient.startTime; + } + + get endTime(): number { + return this.baseClient.endTime; + } + + get duration(): number { + return this.baseClient.duration; + } + + get samplingFrequency(): number { + return this.baseClient.samplingFrequency / this.downsampleFactor; + } + + get numChannels(): number { + return this.baseClient.numChannels; + } + + get numSamples(): number { + return Math.floor(this.baseClient.numSamples / this.downsampleFactor); + } + + get chunkSizeSec(): number { + return this.baseClient.chunkSizeSec; + } + + isLabeledEvents(): boolean { + return this.baseClient.isLabeledEvents(); + } + + getLabels(): string[] | undefined { + return this.baseClient.getLabels(); + } +} diff --git a/vite.config.ts b/vite.config.ts index bb156822..d8e19868 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -5,6 +5,17 @@ import path from 'path' // https://vite.dev/config/ export default defineConfig({ plugins: [react()], + server: { + host: true, // Allow external connections + port: 5173, // Explicit port + hmr: { + overlay: true, // Show errors in overlay + }, + watch: { + usePolling: true, // Use polling for file watching (helps with some file systems) + interval: 1000, // Poll every second + } + }, resolve: { alias: { '@css': path.resolve(__dirname, './src/css'),