Skip to content

Commit 49d4396

Browse files
Peter N. SteinmetzPeter N. Steinmetz
Peter N. Steinmetz
authored and
Peter N. Steinmetz
committed
Replace pattern dictionaries with global compiled patterns.
Now tests for existence of the patterns and produces error or None values as appropriate.
1 parent 44db640 commit 49d4396

File tree

2 files changed

+108
-144
lines changed

2 files changed

+108
-144
lines changed

‎neo/rawio/neuralynxrawio/nlxheader.py‎

Lines changed: 98 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,96 @@ def _to_bool(txt):
7979
("NLX_Base_Class_Type", "", None), # in version 4 and earlier versions of Cheetah
8080
]
8181

82+
# Filename and datetime may appear in header lines starting with # at
83+
# beginning of header or in later versions as a property. The exact format
84+
# used depends on the application name and its version as well as the
85+
# -FileVersion property. Examples of each known case are shown below in comments.
86+
#
87+
# There are now separate patterns for the in header line and in properties cases which cover
88+
# the known variations in each case. They are compiled here once for efficiency.
89+
openDatetime1_pat = re.compile(r"## (Time|Date) Opened:* \((m/d/y|mm/dd/yyy)\): (?P<date>\S+)"\
90+
r"\s+(\(h:m:s\.ms\)|At Time:) (?P<time>\S+)")
91+
openDatetime2_pat = re.compile(r"-TimeCreated (?P<date>\S+) (?P<time>\S+)")
92+
closeDatetime1_pat = re.compile(r"## (Time|Date) Closed:* \((m/d/y|mm/dd/yyy)\): " \
93+
r"(?P<date>\S+)\s+(\(h:m:s\.ms\)|At Time:) (?P<time>\S+)")
94+
closeDatetime2_pat = re.compile(r"-TimeClosed (?P<date>\S+) (?P<time>\S+)")
95+
96+
# BML - example
97+
# ######## Neuralynx Data File Header
98+
# ## File Name: null
99+
# ## Time Opened: (m/d/y): 12/11/15 At Time: 11:37:39.000
100+
101+
# Cheetah after version 1 and before version 5 - example
102+
# ######## Neuralynx Data File Header
103+
# ## File Name F:\2000-01-01_18-28-39\RMH3.ncs
104+
# ## Time Opened (m/d/y): 1/1/2000 (h:m:s.ms) 18:28:39.821
105+
# ## Time Closed (m/d/y): 1/1/2000 (h:m:s.ms) 9:58:41.322
106+
107+
# Cheetah version 4.0.2 - example
108+
# ######## Neuralynx Data File Header
109+
# ## File Name: D:\Cheetah_Data\2003-10-4_10-2-58\CSC14.Ncs
110+
# ## Time Opened: (m/d/y): 10/4/2003 At Time: 10:3:0.578
111+
112+
# Cheetah version 5.4.0 - example
113+
# ######## Neuralynx Data File Header
114+
# ## File Name C:\CheetahData\2000-01-01_00-00-00\CSC5.ncs
115+
# ## Time Opened (m/d/y): 1/01/2001 At Time: 0:00:00.000
116+
# ## Time Closed (m/d/y): 1/01/2001 At Time: 00:00:00.000
117+
118+
# Cheetah version 5.5.1 - example
119+
# ######## Neuralynx Data File Header
120+
# ## File Name C:\CheetahData\2013-11-29_17-05-05\Tet3a.ncs
121+
# ## Time Opened (m/d/y): 11/29/2013 (h:m:s.ms) 17:5:16.793
122+
# ## Time Closed (m/d/y): 11/29/2013 (h:m:s.ms) 18:3:13.603
123+
124+
# Cheetah version 5.6.0 - example
125+
# ## File Name: F:\processing\sum-big-board\252-1375\recording-20180107\2018-01-07_15-14-54\04. tmaze1-no-light-start To tmaze1-light-stop\VT1_fixed.nvt
126+
# ## Time Opened: (m/d/y): 2/5/2018 At Time: 18:5:12.654
127+
128+
# Cheetah version 5.6.3 - example
129+
# ######## Neuralynx Data File Header
130+
# ## File Name C:\CheetahData\2016-11-28_21-50-00\CSC1.ncs
131+
# ## Time Opened (m/d/y): 11/28/2016 (h:m:s.ms) 21:50:33.322
132+
# ## Time Closed (m/d/y): 11/28/2016 (h:m:s.ms) 22:44:41.145
133+
134+
# Cheetah version 5.7.4 - example
135+
# ######## Neuralynx Data File Header
136+
# and then properties
137+
# -OriginalFileName "C:\CheetahData\2017-02-16_17-55-55\CSC1.ncs"
138+
# -TimeCreated 2017/02/16 17:56:04
139+
# -TimeClosed 2017/02/16 18:01:18
140+
141+
# Cheetah version 6.3.2 - example
142+
# ######## Neuralynx Data File Header
143+
# and then properties
144+
# -OriginalFileName "G:\CheetahDataD\2019-07-12_13-21-32\CSC1.ncs"
145+
# -TimeCreated 2019/07/12 13:21:32
146+
# -TimeClosed 2019/07/12 15:07:55
147+
148+
# Cheetah version 6.4.1dev - example
149+
# ######## Neuralynx Data File Header
150+
# and then properties
151+
# -OriginalFileName "D:\CheetahData\2021-02-26_15-46-33\CSC1.ncs"
152+
# -TimeCreated 2021/02/26 15:46:52
153+
# -TimeClosed 2021/10/12 09:07:58
154+
155+
# neuraview version 2 - example
156+
# ######## Neuralynx Data File Header
157+
# ## File Name: L:\McHugh Lab\Recording\2015-06-24_18-05-11\NeuraviewEventMarkers-20151214_SleepScore.nev
158+
# ## Date Opened: (mm/dd/yyy): 12/14/2015 At Time: 15:58:32
159+
# ## Date Closed: (mm/dd/yyy): 12/14/2015 At Time: 15:58:32
160+
161+
# pegasus version 2.1.1 and Cheetah beyond version 5.6.4 - example
162+
# ######## Neuralynx Data File Header
163+
# and then properties
164+
# -OriginalFileName D:\Pegasus Data\Dr_NM\1902\2019-06-28_17-36-50\Events_0008.nev
165+
# -TimeCreated 2019/06/28 17:36:50
166+
# -TimeClosed 2019/06/28 17:45:48
167+
168+
# regular expressions to match filename
169+
filename_regex = [r"## File Name (?P<filename>\S+)",
170+
r'-OriginalFileName "?(?P<filename>\S+)"?']
171+
82172
def __init__(self, filename, props_only=False):
83173
"""
84174
Factory function to build NlxHeader for a given file.
@@ -163,7 +253,7 @@ def setApplicationAndVersion(self):
163253
# BML Ncs file contain neither property, but 'NLX_Base_Class_Type'
164254
elif "NLX_Base_Class_Type" in self:
165255
self["ApplicationName"] = "BML"
166-
app_version = "2.0"
256+
app_version = "1.0"
167257
# Neuraview Ncs file contained neither property nor NLX_Base_Class_Type information
168258
else:
169259
self["ApplicationName"] = "Neuraview"
@@ -204,151 +294,26 @@ def convert_channel_ids_names(self, filename):
204294

205295
return len(chid_entries)
206296

207-
# Filename and datetime may appear in header lines starting with # at
208-
# beginning of header or in later versions as a property. The exact format
209-
# used depends on the application name and its version as well as the
210-
# -FileVersion property. Examples of each known case are shown below in comments.
211-
#
212-
# There are 4 styles understood by this code and the patterns used for parsing
213-
# the items within each are stored in a dictionary. Each dictionary is then
214-
# stored in main dictionary keyed by an abbreviation for the style.
215-
header_pattern_dicts = {
216-
# BML - example
217-
# ######## Neuralynx Data File Header
218-
# ## File Name: null
219-
# ## Time Opened: (m/d/y): 12/11/15 At Time: 11:37:39.000
220-
221-
# Cheetah after version 1 and before version 5 - example
222-
# ######## Neuralynx Data File Header
223-
# ## File Name F:\2000-01-01_18-28-39\RMH3.ncs
224-
# ## Time Opened (m/d/y): 1/1/2000 (h:m:s.ms) 18:28:39.821
225-
# ## Time Closed (m/d/y): 1/1/2000 (h:m:s.ms) 9:58:41.322
226-
227-
# Cheetah version 4.0.2 - example
228-
# ######## Neuralynx Data File Header
229-
# ## File Name: D:\Cheetah_Data\2003-10-4_10-2-58\CSC14.Ncs
230-
# ## Time Opened: (m/d/y): 10/4/2003 At Time: 10:3:0.578
231-
232-
# Cheetah version 5.4.0 - example
233-
# ######## Neuralynx Data File Header
234-
# ## File Name C:\CheetahData\2000-01-01_00-00-00\CSC5.ncs
235-
# ## Time Opened (m/d/y): 1/01/2001 At Time: 0:00:00.000
236-
# ## Time Closed (m/d/y): 1/01/2001 At Time: 00:00:00.000
237-
238-
# Cheetah version 5.5.1 - example
239-
# ######## Neuralynx Data File Header
240-
# ## File Name C:\CheetahData\2013-11-29_17-05-05\Tet3a.ncs
241-
# ## Time Opened (m/d/y): 11/29/2013 (h:m:s.ms) 17:5:16.793
242-
# ## Time Closed (m/d/y): 11/29/2013 (h:m:s.ms) 18:3:13.603
243-
244-
# Cheetah version 5.6.0 - example
245-
# ## File Name: F:\processing\sum-big-board\252-1375\recording-20180107\2018-01-07_15-14-54\04. tmaze1-no-light-start To tmaze1-light-stop\VT1_fixed.nvt
246-
# ## Time Opened: (m/d/y): 2/5/2018 At Time: 18:5:12.654
247-
248-
# Cheetah version 5.6.3 - example
249-
# ######## Neuralynx Data File Header
250-
# ## File Name C:\CheetahData\2016-11-28_21-50-00\CSC1.ncs
251-
# ## Time Opened (m/d/y): 11/28/2016 (h:m:s.ms) 21:50:33.322
252-
# ## Time Closed (m/d/y): 11/28/2016 (h:m:s.ms) 22:44:41.145
253-
254-
# Cheetah version 5.7.4 - example
255-
# ######## Neuralynx Data File Header
256-
# and then properties
257-
# -OriginalFileName "C:\CheetahData\2017-02-16_17-55-55\CSC1.ncs"
258-
# -TimeCreated 2017/02/16 17:56:04
259-
# -TimeClosed 2017/02/16 18:01:18
260-
261-
# Cheetah version 6.3.2 - example
262-
# ######## Neuralynx Data File Header
263-
# and then properties
264-
# -OriginalFileName "G:\CheetahDataD\2019-07-12_13-21-32\CSC1.ncs"
265-
# -TimeCreated 2019/07/12 13:21:32
266-
# -TimeClosed 2019/07/12 15:07:55
267-
268-
# Cheetah version 6.4.1dev - example
269-
# ######## Neuralynx Data File Header
270-
# and then properties
271-
# -OriginalFileName "D:\CheetahData\2021-02-26_15-46-33\CSC1.ncs"
272-
# -TimeCreated 2021/02/26 15:46:52
273-
# -TimeClosed 2021/10/12 09:07:58
274-
275-
# neuraview version 2 - example
276-
# ######## Neuralynx Data File Header
277-
# ## File Name: L:\McHugh Lab\Recording\2015-06-24_18-05-11\NeuraviewEventMarkers-20151214_SleepScore.nev
278-
# ## Date Opened: (mm/dd/yyy): 12/14/2015 At Time: 15:58:32
279-
# ## Date Closed: (mm/dd/yyy): 12/14/2015 At Time: 15:58:32
280-
281-
# pegasus version 2.1.1 and Cheetah beyond version 5.6.4 - example
282-
# ######## Neuralynx Data File Header
283-
# and then properties
284-
# -OriginalFileName D:\Pegasus Data\Dr_NM\1902\2019-06-28_17-36-50\Events_0008.nev
285-
# -TimeCreated 2019/06/28 17:36:50
286-
# -TimeClosed 2019/06/28 17:45:48
287-
288-
"combined": dict(
289-
openDatetime1_regex=r"## (Time|Date) Opened:* \((m/d/y|mm/dd/yyy)\): (?P<date>\S+)" \
290-
r"\s+(\(h:m:s\.ms\)|At Time:) (?P<time>\S+)",
291-
openDatetime2_regex=r"-TimeCreated (?P<date>\S+) (?P<time>\S+)",
292-
closeDatetime1_regex=r"## (Time|Date) Closed:* \((m/d/y|mm/dd/yyy)\): (?P<date>\S+)" \
293-
r"\s+(\(h:m:s\.ms\)|At Time:) (?P<time>\S+)",
294-
closeDatetime2_regex=r"-TimeClosed (?P<date>\S+) (?P<time>\S+)",
295-
)
296-
}
297-
298-
# regular expressions to match filename
299-
filename_regex = [r"## File Name (?P<filename>\S+)",
300-
r'-OriginalFileName "?(?P<filename>\S+)"?']
301-
302297
def readTimeDate(self, txt_header):
303298
"""
304-
Read time and date from text of header appropriate for app name and version
305-
306-
:TODO: this works for current examples but is not likely actually related
307-
to app version in this manner.
299+
Read time and date from text of header
308300
"""
309-
an = self["ApplicationName"]
310-
if an == "Cheetah":
311-
av = self["ApplicationVersion"]
312-
if av <= Version("2"): # version 1 uses same as older versions
313-
hpd = NlxHeader.header_pattern_dicts["combined"]
314-
elif av < Version("5"):
315-
hpd = NlxHeader.header_pattern_dicts["combined"]
316-
elif av <= Version("5.4.0"):
317-
hpd = NlxHeader.header_pattern_dicts["combined"]
318-
elif av == Version("5.6.0"):
319-
hpd = NlxHeader.header_pattern_dicts["combined"]
320-
elif av <= Version("5.6.4"):
321-
hpd = NlxHeader.header_pattern_dicts["combined"]
322-
else:
323-
hpd = NlxHeader.header_pattern_dicts["combined"]
324-
elif an == "BML":
325-
hpd = NlxHeader.header_pattern_dicts["combined"]
326-
av = Version("2")
327-
elif an == "Neuraview":
328-
hpd = NlxHeader.header_pattern_dicts["combined"]
329-
av = Version("2")
330-
elif an == "Pegasus":
331-
hpd = NlxHeader.header_pattern_dicts["combined"]
332-
av = Version("2")
333-
else:
334-
an = "Unknown"
335-
av = "NA"
336-
hpd = NlxHeader.header_pattern_dicts["combined"]
337301

338302
# opening time
339-
sr = re.search(hpd["openDatetime1_regex"], txt_header)
340-
if not sr: sr=re.search(hpd["openDatetime2_regex"], txt_header)
303+
sr = NlxHeader.openDatetime1_pat.search(txt_header)
304+
if not sr: sr=NlxHeader.openDatetime2_pat.search(txt_header)
341305
if not sr:
342306
raise IOError(
343-
f"No matching header open date/time for application {an} " + f"version {av}. Please contact developers."
307+
f"No matching header open date/time for application {self['ApplicationName']} " +
308+
f"version {self['ApplicationVersion']}. Please contact developers."
344309
)
345310
else:
346311
dt1 = sr.groupdict()
347312
self['recording_opened'] = dateutil.parser.parse(f"{dt1['date']} {dt1['time']}")
348313

349314
# close time, if available
350-
sr = re.search(hpd["closeDatetime1_regex"], txt_header)
351-
if not sr: sr=re.search(hpd["closeDatetime2_regex"], txt_header)
315+
sr = NlxHeader.closeDatetime1_pat.search(txt_header)
316+
if not sr: sr=NlxHeader.closeDatetime2_pat.search(txt_header)
352317
if sr:
353318
dt2 = sr.groupdict()
354319
self['recording_closed'] = dateutil.parser.parse(f"{dt2['date']} {dt2['time']}")

‎neo/test/rawiotest/test_neuralynxrawio.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -385,22 +385,21 @@ def test_neuraview2(self):
385385
self.assertEqual(datetime.datetime(2015,12,14, 15,58,32), hdr['recording_opened'])
386386
self.assertEqual(datetime.datetime(2015,12,14, 15,58,32), hdr['recording_closed'])
387387

388+
# left in for possible future header tests
388389
def get_text_header(self, filename):
389390
with open(filename, "rb") as f:
390391
txt_header = f.read(NlxHeader.HEADER_SIZE)
391392
return txt_header.strip(b"\x00").decode("latin-1")
392393

393-
def check_dateutil_parse(self, hdrTxt, hdrPatternName, openDate, closeDate):
394-
hpd = NlxHeader.header_pattern_dicts[hdrPatternName]
395-
mtch = re.search(hpd["openDatetime1_regex"], hdrTxt)
396-
if mtch is None: mtch = re.search(hpd["openDatetime2_regex"], hdrTxt)
394+
# left in for possible future header tests
395+
def check_dateutil_parse(self, hdrTxt, openPat, closePat, openDate, closeDate):
396+
mtch = openPat.search(hdrTxt)
397397
self.assertIsNotNone(mtch)
398398
dt = mtch.groupdict()
399399
date = dateutil.parser.parse(f"{dt['date']} {dt['time']}")
400400
self.assertEqual(openDate, date)
401-
if closeDate is not None:
402-
mtch = re.search(hpd["closeDatetime1_regex"], hdrTxt)
403-
if mtch is None: mtch = re.search(hpd["closeDatetime2_regex"], hdrTxt)
401+
if closePat is not None:
402+
mtch = closePat.search(hdrTxt)
404403
self.assertIsNotNone(mtch)
405404
dt = mtch.groupdict()
406405
date = dateutil.parser.parse(f"{dt['date']} {dt['time']}")
@@ -411,7 +410,7 @@ def test_datetime_parsing(self):
411410
# neuraview2
412411
filename = self.get_local_path("neuralynx/Neuraview_v2/original_data/NeuraviewEventMarkers-sample.nev")
413412
txt_header = self.get_text_header(filename)
414-
self.check_dateutil_parse(txt_header, 'combined',
413+
self.check_dateutil_parse(txt_header, NlxHeader.openDatetime1_pat, NlxHeader.closeDatetime1_pat,
415414
datetime.datetime(2015,12,14, 15,58,32),
416415
datetime.datetime(2015,12,14, 15,58,32))
417416
hdr = NlxHeader(filename)
@@ -423,7 +422,7 @@ def test_datetime_parsing(self):
423422
# Cheetah 5.7.4 'inProps'
424423
filename = self.get_local_path("neuralynx/Cheetah_v5.7.4/original_data/CSC1.ncs")
425424
txt_header = self.get_text_header(filename)
426-
self.check_dateutil_parse(txt_header, 'combined',
425+
self.check_dateutil_parse(txt_header, NlxHeader.openDatetime2_pat, NlxHeader.closeDatetime2_pat,
427426
datetime.datetime(2017,2,16, 17,56,4),
428427
datetime.datetime(2017,2,16, 18,1,18))
429428
hdr = NlxHeader(filename)
@@ -435,7 +434,7 @@ def test_datetime_parsing(self):
435434
# Cheetah 4.0.2
436435
filename = self.get_local_path("neuralynx/Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs")
437436
txt_header = self.get_text_header(filename)
438-
self.check_dateutil_parse(txt_header, 'combined',
437+
self.check_dateutil_parse(txt_header, NlxHeader.openDatetime1_pat, None,
439438
datetime.datetime(2003,10,4, 10,3,0, 578000),
440439
None)
441440
hdr = NlxHeader(filename)
@@ -446,7 +445,7 @@ def test_datetime_parsing(self):
446445
# Cheetah 5.4.0 'openClosedInHeader'
447446
filename = self.get_local_path("neuralynx/Cheetah_v5.4.0/original_data/CSC5_trunc.Ncs")
448447
txt_header = self.get_text_header(filename)
449-
self.check_dateutil_parse(txt_header, 'combined',
448+
self.check_dateutil_parse(txt_header, NlxHeader.openDatetime1_pat, NlxHeader.closeDatetime1_pat,
450449
datetime.datetime(2001,1,1, 0,0,0, 0),
451450
datetime.datetime(2001,1,1, 0,0,0, 0))
452451
hdr = NlxHeader(filename)

0 commit comments

Comments
 (0)