|
| 1 | +""" |
| 2 | +NeuroNexus has their own file format based on their Allego Recording System |
| 3 | +https://www.neuronexus.com/webinar/allego-software-updates/ |
| 4 | +
|
| 5 | +The format involves 3 files: |
| 6 | + * The *.xdat.json metadata file |
| 7 | + * The *_data.xdat binary file of all raw data |
| 8 | + * The *_timestamps.xdat binary file of the timestamp data |
| 9 | + |
| 10 | +Based on sample data is appears that the binary file is always a float32 format |
| 11 | +Other information can be found within the metadata json file |
| 12 | +
|
| 13 | +
|
| 14 | +The metadata file has a pretty complicated structure as far as I can tell |
| 15 | +a lot of which is dedicated to probe information, which won't be handle at the |
| 16 | +the Neo level. |
| 17 | +
|
| 18 | +It appears that the metadata['status'] provides most of the information necessary |
| 19 | +for generating the initial memory map (global sampling frequency), n_channels, |
| 20 | +n_samples. |
| 21 | +
|
| 22 | +metadata['sapiens_base']['biointerface_map'] provides all the channel specific information |
| 23 | +like channel_names, channel_ids, channel_types. |
| 24 | +
|
| 25 | +An additional note on channels. It appears that analog channels are called `pri` or |
| 26 | +`ai0` within the metadata whereas digital channels are called `din0` or `dout0`. |
| 27 | +In this first implementation it is up to the user to do the appropriate channel slice |
| 28 | +to only get the data they want. This is a buffer-based approach that Sam likes. |
| 29 | +Eventually we will try to divide these channels into streams (analog vs digital) or |
| 30 | +we can come up with a work around if users open an issue requesting this. |
| 31 | +
|
| 32 | +Zach McKenzie |
| 33 | +
|
| 34 | +""" |
| 35 | + |
| 36 | +from __future__ import annotations |
| 37 | +from pathlib import Path |
| 38 | +import json |
| 39 | +import datetime |
| 40 | + |
| 41 | +import numpy as np |
| 42 | + |
| 43 | +from .baserawio import ( |
| 44 | + BaseRawIO, |
| 45 | + _signal_channel_dtype, |
| 46 | + _signal_stream_dtype, |
| 47 | + _spike_channel_dtype, |
| 48 | + _event_channel_dtype, |
| 49 | +) |
| 50 | +from neo.core import NeoReadWriteError |
| 51 | + |
| 52 | + |
| 53 | +class NeuroNexusRawIO(BaseRawIO): |
| 54 | + |
| 55 | + extensions = ["xdat", "json"] |
| 56 | + rawmode = "one-file" |
| 57 | + |
| 58 | + def __init__(self, filename: str | Path = ""): |
| 59 | + """ |
| 60 | + The Allego NeuroNexus reader for the `xdat` file format |
| 61 | +
|
| 62 | + Parameters |
| 63 | + ---------- |
| 64 | + filename: str | Path, default: '' |
| 65 | + The filename of the metadata file should end in .xdat.json |
| 66 | +
|
| 67 | + Notes |
| 68 | + ----- |
| 69 | + * The format involves 3 files: |
| 70 | + * The *.xdat.json metadata file |
| 71 | + * The *_data.xdat binary file of all raw data |
| 72 | + * The *_timestamps.xdat binary file of the timestamp data |
| 73 | + From the metadata the other two files are located within the same directory |
| 74 | + and loaded. |
| 75 | +
|
| 76 | + * The metadata is stored as the metadata attribute for individuals hoping |
| 77 | + to extract probe information, but Neo itself does not load any of the probe |
| 78 | + information |
| 79 | +
|
| 80 | + Examples |
| 81 | + -------- |
| 82 | + >>> from neo.rawio import NeuronexusRawIO |
| 83 | + >>> reader = NeuronexusRawIO(filename='abc.xdat.json') |
| 84 | + >>> reader.parse_header() |
| 85 | + >>> raw_chunk = reader.get_analogsignal_chunk(block_index=0 |
| 86 | + seg_index=0, |
| 87 | + stream_index=0) |
| 88 | + # this isn't necessary for this reader since data is stored as float uV, but |
| 89 | + # this is included in case there is a future change to the format |
| 90 | + >>> float_chunk = reader.rescale_signal_raw_to_float(raw_chunk, stream_index=0) |
| 91 | +
|
| 92 | + """ |
| 93 | + |
| 94 | + BaseRawIO.__init__(self) |
| 95 | + |
| 96 | + if not Path(filename).is_file(): |
| 97 | + raise FileNotFoundError(f"The metadata file {filename} was not found") |
| 98 | + if Path(filename).suffix != ".json": |
| 99 | + raise NeoReadWriteError( |
| 100 | + f"The json metadata file should be given, filename entered is {Path(filename).stem}" |
| 101 | + ) |
| 102 | + meta_filename = Path(filename) |
| 103 | + binary_file = meta_filename.parent / (meta_filename.stem.split(".")[0] + "_data.xdat") |
| 104 | + |
| 105 | + if not binary_file.exists() and not binary_file.is_file(): |
| 106 | + raise FileNotFoundError(f"The data.xdat file {binary_file} was not found. Is it in the same directory?") |
| 107 | + timestamp_file = meta_filename.parent / (meta_filename.stem.split(".")[0] + "_timestamp.xdat") |
| 108 | + if not timestamp_file.exists() and not timestamp_file.is_file(): |
| 109 | + raise FileNotFoundError( |
| 110 | + f"The timestamps.xdat file {timestamp_file} was not found. Is it in the same directory?" |
| 111 | + ) |
| 112 | + |
| 113 | + self.filename = filename |
| 114 | + self.binary_file = binary_file |
| 115 | + self.timestamp_file = timestamp_file |
| 116 | + |
| 117 | + def _source_name(self): |
| 118 | + # return the metadata filename only |
| 119 | + return self.filename |
| 120 | + |
| 121 | + def _parse_header(self): |
| 122 | + |
| 123 | + # read metadata |
| 124 | + self.metadata = self.read_metadata(self.filename) |
| 125 | + |
| 126 | + # Collect information necessary for memory map |
| 127 | + self._sampling_frequency = self.metadata["status"]["samp_freq"] |
| 128 | + self._n_samples, self._n_channels = self.metadata["status"]["shape"] |
| 129 | + # Stored as a simple float32 binary file |
| 130 | + BINARY_DTYPE = "float32" |
| 131 | + binary_file = self.binary_file |
| 132 | + timestamp_file = self.timestamp_file |
| 133 | + |
| 134 | + # Make the two memory maps |
| 135 | + self._raw_data = np.memmap( |
| 136 | + binary_file, |
| 137 | + dtype=BINARY_DTYPE, |
| 138 | + mode="r", |
| 139 | + shape=(self._n_samples, self._n_channels), |
| 140 | + offset=0, # headerless binary file |
| 141 | + ) |
| 142 | + self._timestamps = np.memmap( |
| 143 | + timestamp_file, |
| 144 | + dtype=np.int64, # this is from the allego sample reader timestamps are np.int64 |
| 145 | + mode="r", |
| 146 | + offset=0, # headerless binary file |
| 147 | + ) |
| 148 | + |
| 149 | + # We can do a quick timestamp check to make sure it is the correct timestamp data for the |
| 150 | + # given metadata |
| 151 | + if self._timestamps[0] != self.metadata["status"]["timestamp_range"][0]: |
| 152 | + metadata_start = self.metadata["status"]["timestamp_range"][0] |
| 153 | + data_start = self._teimstamps[0] |
| 154 | + raise NeoReadWriteError( |
| 155 | + f"The metadata indicates a different starting timestamp {metadata_start} than the data starting timestamp {data_start}" |
| 156 | + ) |
| 157 | + |
| 158 | + # organize the channels |
| 159 | + signal_channels = [] |
| 160 | + channel_info = self.metadata["sapiens_base"]["biointerface_map"] |
| 161 | + |
| 162 | + # as per dicussion with the Neo/SpikeInterface teams stream_id will become buffer_id |
| 163 | + # and because all data is stored in the same buffer stream for the moment all channels |
| 164 | + # will be in stream_id = 0. In the future this will be split into sub_streams based on |
| 165 | + # type but for now it will be the end-users responsability for this. |
| 166 | + stream_id = '0' # hard-coded see note above |
| 167 | + for channel_index, channel_name in enumerate(channel_info["chan_name"]): |
| 168 | + channel_id = channel_info["ntv_chan_name"][channel_index] |
| 169 | + # 'ai0' indicates analog data which is stored as microvolts |
| 170 | + if channel_info["chan_type"][channel_index] == "ai0": |
| 171 | + units = "uV" |
| 172 | + # 'd' means digital. Per discussion with neuroconv users the use of |
| 173 | + # 'a.u.' makes the units clearer |
| 174 | + elif channel_info["chan_type"][channel_index][0] == "d": |
| 175 | + units = "a.u." |
| 176 | + # aux channel |
| 177 | + else: |
| 178 | + units = "V" |
| 179 | + |
| 180 | + signal_channels.append( |
| 181 | + ( |
| 182 | + channel_name, |
| 183 | + channel_id, |
| 184 | + self._sampling_frequency, |
| 185 | + BINARY_DTYPE, |
| 186 | + units, |
| 187 | + 1, # no gain |
| 188 | + 0, # no offset |
| 189 | + stream_id, |
| 190 | + ) |
| 191 | + ) |
| 192 | + |
| 193 | + signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) |
| 194 | + |
| 195 | + stream_ids = np.unique(signal_channels["stream_id"]) |
| 196 | + signal_streams = np.zeros(stream_ids.size, dtype=_signal_stream_dtype) |
| 197 | + signal_streams["id"] = [str(stream_id) for stream_id in stream_ids] |
| 198 | + for stream_index, stream_id in enumerate(stream_ids): |
| 199 | + name = stream_id_to_stream_name.get(int(stream_id), "") |
| 200 | + signal_streams["name"][stream_index] = name |
| 201 | + |
| 202 | + # No events |
| 203 | + event_channels = [] |
| 204 | + event_channels = np.array(event_channels, dtype=_event_channel_dtype) |
| 205 | + |
| 206 | + # No spikes |
| 207 | + spike_channels = [] |
| 208 | + spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype) |
| 209 | + |
| 210 | + # Put all the necessary info in the header |
| 211 | + self.header = {} |
| 212 | + self.header["nb_block"] = 1 |
| 213 | + self.header["nb_segment"] = [1] |
| 214 | + self.header["signal_streams"] = signal_streams |
| 215 | + self.header["signal_channels"] = signal_channels |
| 216 | + self.header["spike_channels"] = spike_channels |
| 217 | + self.header["event_channels"] = event_channels |
| 218 | + |
| 219 | + # Add the minimum annotations |
| 220 | + self._generate_minimal_annotations() |
| 221 | + |
| 222 | + # date comes out as: |
| 223 | + # year-month-daydayofweektime all as a string so we need to prep it for |
| 224 | + # entering into datetime |
| 225 | + # example: '2024-07-01T13:04:49.4972245-04:00' |
| 226 | + stringified_date_list = self.metadata['status']['start_time'].split('-') |
| 227 | + year = int(stringified_date_list[0]) |
| 228 | + month = int(stringified_date_list[1]) |
| 229 | + day = int(stringified_date_list[2][:2]) # day should be first two digits of the third item in list |
| 230 | + time_info = stringified_date_list[2].split(':') |
| 231 | + hour = int(time_info[0][-2:]) |
| 232 | + minute = int(time_info[1]) |
| 233 | + second = int(float(time_info[2])) |
| 234 | + microsecond = int(1000 * 1000 * (float(time_info[2]) - second))# second -> micro is 1000 * 1000 |
| 235 | + |
| 236 | + rec_datetime = datetime.datetime(year, month, day, hour, minute, second, microsecond) |
| 237 | + bl_annotations = self.raw_annotations["blocks"][0] |
| 238 | + seg_annotations = bl_annotations["segments"][0] |
| 239 | + for d in (bl_annotations, seg_annotations): |
| 240 | + d["rec_datetime"] = rec_datetime |
| 241 | + |
| 242 | + def _get_signal_size(self, block_index, seg_index, stream_index): |
| 243 | + |
| 244 | + # All streams have the same size so just return the raw_data size |
| 245 | + return self._raw_data.size |
| 246 | + |
| 247 | + def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): |
| 248 | + |
| 249 | + if i_start is None: |
| 250 | + i_start = 0 |
| 251 | + if i_stop is None: |
| 252 | + i_stop = self._get_signal_size(block_index, seg_index, stream_index) |
| 253 | + |
| 254 | + raw_data = self._raw_data[i_start:i_stop, :] |
| 255 | + |
| 256 | + if channel_indexes is None: |
| 257 | + channel_indexes = slice(None) |
| 258 | + |
| 259 | + raw_data = raw_data[:, channel_indexes] |
| 260 | + return raw_data |
| 261 | + |
| 262 | + def _segment_t_stop(self, block_index, seg_index): |
| 263 | + |
| 264 | + t_stop = self.metadata["status"]["t_range"][1] |
| 265 | + return t_stop |
| 266 | + |
| 267 | + def _segment_t_start(self, block_index, seg_index): |
| 268 | + |
| 269 | + t_start = self.metadata["status"]["t_range"][0] |
| 270 | + return t_start |
| 271 | + |
| 272 | + def _get_signal_t_start(self, block_index, seg_index, stream_index): |
| 273 | + |
| 274 | + t_start = self.metadata["status"]["t_range"][0] |
| 275 | + return t_start |
| 276 | + |
| 277 | + ####################################### |
| 278 | + # Helper Functions |
| 279 | + |
| 280 | + def read_metadata(self, fname_metadata): |
| 281 | + """ |
| 282 | + Metadata is just a heavily nested json file |
| 283 | +
|
| 284 | + Parameters |
| 285 | + ---------- |
| 286 | + fname_metada: str | Path |
| 287 | + The *.xdat.json file for the current recording |
| 288 | +
|
| 289 | + Returns |
| 290 | + ------- |
| 291 | + metadata: dict |
| 292 | + Returns the metadata as a dictionary""" |
| 293 | + |
| 294 | + fname_metadata = Path(fname_metadata) |
| 295 | + with open(fname_metadata, "rb") as read_file: |
| 296 | + metadata = json.load(read_file) |
| 297 | + |
| 298 | + return metadata |
| 299 | + |
| 300 | + |
| 301 | +# this is pretty useless right now, but I think after a |
| 302 | +# refactor with sub streams we could adapt this for the sub-streams |
| 303 | +# so let's leave this here for now :) |
| 304 | +stream_id_to_stream_name = {'0': "Neuronexus Allego Data"} |
0 commit comments