Skip to content

Commit 16b3676

Browse files
samuelgarciasamuelgarcia
committed
Start dataset download with datalad
1 parent 072adfd commit 16b3676

File tree

13 files changed

+335
-211
lines changed

13 files changed

+335
-211
lines changed

‎.circleci/config.yml‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

‎.github/workflows/full-test.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: Full spikeinterface tests
2+
3+
on:
4+
pull_request:
5+
branches: [master]
6+
types: [synchronize, opened, reopened, ready_for_review]
7+
8+
jobs:
9+
build-and-test:
10+
name: Test on (${{ matrix.os }})
11+
runs-on: ${{ matrix.os }}
12+
strategy:
13+
fail-fast: false
14+
matrix:
15+
# "macos-latest", "windows-latest"
16+
os: ["ubuntu-latest", ]
17+
steps:
18+
- uses: actions/checkout@v2
19+
with:
20+
python-version: 3.8
21+
- name: Which python
22+
run: |
23+
which python
24+
- uses: actions/cache@v2
25+
id: cache-venv
26+
with:
27+
path: ./test_env
28+
key: $venv-{{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements_testing.txt') }}
29+
restore-keys: |
30+
${{ runner.os }}-venv-
31+
- name: Install dependencies
32+
run: |
33+
# this is for datalad and download testing datasets
34+
sudo apt install git git-annex
35+
# needed for correct operation of git/git-annex/DataLad
36+
git config --global user.email "neo_ci@fake_mail.com"
37+
git config --global user.name "neo CI"
38+
# create an environement and install everything
39+
python -m venv test_env
40+
source ./test_env/bin/activate
41+
python -m pip install --upgrade pip
42+
pip install -r requirements.txt
43+
pip install -r requirements_testing.txt
44+
pip install -e .
45+
- uses: actions/cache@v2
46+
id: cache-datasets
47+
env:
48+
# the key depend on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
49+
HASH_EPHY_DATASET: git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1
50+
with:
51+
path: ~/ephy_testing_data
52+
key: ${{ runner.os }}-datasets-${HASH_EPHY_DATASET}
53+
restore-keys: |
54+
${{ runner.os }}-datasets
55+
- name: Test with pytest
56+
run: |
57+
source ./test_env/bin/activate
58+
pytest --cov=neo neo/tests

‎neo/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
logging_handler = logging.StreamHandler()
99

1010
from neo.core import *
11-
# ~ import neo.rawio
11+
# import neo.rawio
1212
from neo.io import *
1313

1414
from neo.version import version as __version__

‎neo/rawio/baserawio.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
('offset', 'float64'),
7777
('stream_id', 'U64'),
7878
]
79+
7980
# TODO for later: add t_start and length in _signal_channel_dtype
8081
# this would simplify all t_start/t_stop stuff for each RawIO class
8182

‎neo/test/rawiotest/common_rawio_test.py‎

Lines changed: 83 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@
2121
import logging
2222
import unittest
2323

24-
from neo.test.rawiotest.tools import (can_use_network, make_all_directories,
25-
download_test_file, create_local_temp_dir)
24+
#~ from neo.test.rawiotest.tools import (can_use_network, make_all_directories,
25+
#~ download_test_file, create_local_temp_dir)
26+
from neo.utils import download_dataset, get_local_testing_data_folder
27+
28+
from neo.test.rawiotest.tools import can_use_network
2629

2730
from neo.test.rawiotest import rawio_compliance as compliance
2831

2932

3033
# url_for_tests = "https://portal.g-node.org/neo/" #This is the old place
31-
url_for_tests = "https://web.gin.g-node.org/NeuralEnsemble/ephy_testing_data/raw/master/"
32-
34+
#~ url_for_tests = "https://web.gin.g-node.org/NeuralEnsemble/ephy_testing_data/raw/master/"
35+
repo_for_test = 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data'
3336

3437
class BaseTestRawIO:
3538
'''
@@ -44,9 +47,9 @@ class BaseTestRawIO:
4447
# all IO test need to modify this:
4548
rawioclass = None # the IOclass to be tested
4649

47-
files_to_test = [] # list of files to test compliances
48-
files_to_download = [] # when files are at G-Node
49-
50+
entities_to_test = [] # list of files to test compliances
51+
entities_to_download = [] # when files are at gin
52+
5053
# allow environment to tell avoid using network
5154
use_network = can_use_network()
5255

@@ -57,60 +60,88 @@ def setUp(self):
5760
Set up the test fixture. This is run for every test
5861
'''
5962
self.shortname = self.rawioclass.__name__.lower().replace('rawio', '')
60-
self.create_local_dir_if_not_exists()
61-
self.download_test_files_if_not_present()
62-
63-
def create_local_dir_if_not_exists(self):
64-
'''
65-
Create a local directory to store testing files and return it.
66-
67-
The directory path is also written to self.local_test_dir
68-
'''
69-
self.local_test_dir = create_local_temp_dir(self.shortname)
70-
return self.local_test_dir
71-
72-
def download_test_files_if_not_present(self):
73-
'''
74-
Download %s file at G-node for testing
75-
url_for_tests is global at beginning of this file.
76-
77-
''' % self.rawioclass.__name__
78-
79-
if not self.use_network:
80-
raise unittest.SkipTest("Requires download of data from the web")
81-
82-
url = url_for_tests + self.shortname
83-
try:
84-
make_all_directories(self.files_to_download, self.local_test_dir)
85-
download_test_file(self.files_to_download,
86-
self.local_test_dir, url)
87-
except OSError as exc:
88-
raise unittest.SkipTest(exc)
89-
90-
download_test_files_if_not_present.__test__ = False
91-
92-
def cleanup_file(self, path):
93-
'''
94-
Remove test files or directories safely.
95-
'''
96-
cleanup_test_file(self.rawioclass, path, directory=self.local_test_dir)
97-
63+
64+
for remote_path in self.entities_to_download:
65+
download_dataset(repo=repo_for_test, remote_path=remote_path)
66+
67+
#~ self.create_local_dir_if_not_exists()
68+
#~ self.download_test_files_if_not_present()
69+
70+
#~ def create_local_dir_if_not_exists(self):
71+
#~ '''
72+
#~ Create a local directory to store testing files and return it.
73+
74+
#~ The directory path is also written to self.local_test_dir
75+
#~ '''
76+
#~ self.local_test_dir = create_local_temp_dir(self.shortname)
77+
#~ return self.local_test_dir
78+
79+
#~ def download_test_files_if_not_present(self):
80+
#~ '''
81+
#~ Download %s file at G-node for testing
82+
#~ url_for_tests is global at beginning of this file.
83+
84+
#~ ''' % self.rawioclass.__name__
85+
86+
#~ if not self.use_network:
87+
#~ raise unittest.SkipTest("Requires download of data from the web")
88+
89+
#~ url = url_for_tests + self.shortname
90+
#~ try:
91+
#~ make_all_directories(self.files_to_download, self.local_test_dir)
92+
#~ download_test_file(self.files_to_download,
93+
#~ self.local_test_dir, url)
94+
#~ except OSError as exc:
95+
#~ raise unittest.SkipTest(exc)
96+
97+
#~ download_test_files_if_not_present.__test__ = False
98+
99+
#~ def cleanup_file(self, path):
100+
#~ '''
101+
#~ Remove test files or directories safely.
102+
#~ '''
103+
#~ cleanup_test_file(self.rawioclass, path, directory=self.local_test_dir)
104+
105+
#~ def get_filename_path(self, filename):
106+
#~ '''
107+
#~ Get the path to a filename in the current temporary file directory
108+
#~ '''
109+
#~ return os.path.join(self.local_test_dir, filename)
110+
111+
def get_local_base_folder(self):
112+
return get_local_testing_data_folder()
113+
114+
def get_local_path(self, sub_path):
115+
root_local_path = self.get_local_base_folder()
116+
local_path = root_local_path / sub_path
117+
# TODO later : remove the str when all IOs handle the Path stuff
118+
local_path = str(local_path)
119+
return local_path
120+
98121
def get_filename_path(self, filename):
99-
'''
100-
Get the path to a filename in the current temporary file directory
101-
'''
102-
return os.path.join(self.local_test_dir, filename)
122+
# keep for backward compatibility
123+
# will be removed soon
124+
classname = self.__class__.__name__
125+
classname = classname.replace('Test', '').replace('RawIO', '').lower()
126+
root_local_path = self.get_local_base_folder()
127+
local_path = root_local_path / classname / filename
128+
# TODO later : remove the str when all IOs handle the Path stuff
129+
local_path = str(local_path)
130+
print('get_filename_path will be removed (use get_local_path() instead)', self.__class__.__name__, local_path)
131+
return local_path
103132

104133
def test_read_all(self):
105134
# Read all file in self.entities_to_test
106135

107136
for entity_name in self.entities_to_test:
108-
entity_name = self.get_filename_path(entity_name)
137+
# entity_name = self.get_filename_path(entity_name)
138+
# local path is a folder or a file
139+
local_path = self.get_local_path(entity_name)
109140

110141
if self.rawioclass.rawmode.endswith('-file'):
111-
reader = self.rawioclass(filename=entity_name)
142+
reader = self.rawioclass(filename=local_path)
112143
elif self.rawioclass.rawmode.endswith('-dir'):
113-
reader = self.rawioclass(dirname=entity_name)
144+
reader = self.rawioclass(dirname=local_path)
114145

115146
txt = reader.__repr__()
116147
assert 'nb_block' not in txt, 'Before parser_header() nb_block should be NOT known'

‎neo/test/rawiotest/test_blackrockrawio.py‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -20,20 +20,24 @@
2020

2121
class TestBlackrockRawIO(BaseTestRawIO, unittest.TestCase, ):
2222
rawioclass = BlackrockRawIO
23-
entities_to_test = ['FileSpec2.3001',
24-
'blackrock_2_1/l101210-001']
25-
26-
files_to_download = [
27-
'FileSpec2.3001.nev',
28-
'FileSpec2.3001.ns5',
29-
'FileSpec2.3001.ccf',
30-
'FileSpec2.3001.mat',
31-
'blackrock_2_1/l101210-001.mat',
32-
'blackrock_2_1/l101210-001_nev-02_ns5.mat',
33-
'blackrock_2_1/l101210-001.ns2',
34-
'blackrock_2_1/l101210-001.ns5',
35-
'blackrock_2_1/l101210-001.nev',
36-
'blackrock_2_1/l101210-001-02.nev']
23+
entities_to_test = [
24+
'blackrock/FileSpec2.3001',
25+
'blackrock/blackrock_2_1/l101210-001'
26+
]
27+
28+
entities_to_download = [
29+
'blackrock'
30+
]
31+
#~ 'FileSpec2.3001.nev',
32+
#~ 'FileSpec2.3001.ns5',
33+
#~ 'FileSpec2.3001.ccf',
34+
#~ 'FileSpec2.3001.mat',
35+
#~ 'blackrock_2_1/l101210-001.mat',
36+
#~ 'blackrock_2_1/l101210-001_nev-02_ns5.mat',
37+
#~ 'blackrock_2_1/l101210-001.ns2',
38+
#~ 'blackrock_2_1/l101210-001.ns5',
39+
#~ 'blackrock_2_1/l101210-001.nev',
40+
#~ 'blackrock_2_1/l101210-001-02.nev']
3741

3842
@unittest.skipUnless(HAVE_SCIPY, "requires scipy")
3943
def test_compare_blackrockio_with_matlabloader(self):
@@ -50,7 +54,7 @@ def test_compare_blackrockio_with_matlabloader(self):
5054
"""
5155

5256
# Load data from Matlab generated files
53-
ml = scipy.io.loadmat(self.get_filename_path('FileSpec2.3001.mat'))
57+
ml = scipy.io.loadmat(self.get_local_path('blackrock/FileSpec2.3001.mat'))
5458

5559
lfp_ml = ml['lfp'] # (channel x time) LFP matrix
5660
ts_ml = ml['ts'] # spike time stamps
@@ -62,7 +66,7 @@ def test_compare_blackrockio_with_matlabloader(self):
6266

6367
# Load data in channels 1-3 from original data files using the Neo
6468
# BlackrockIO
65-
reader = BlackrockRawIO(filename=self.get_filename_path('FileSpec2.3001'))
69+
reader = BlackrockRawIO(filename=self.get_local_path('blackrock/FileSpec2.3001'))
6670
reader.parse_header()
6771

6872
# Check if analog data on channels 1-8 are equal
@@ -124,14 +128,14 @@ def test_compare_blackrockio_with_matlabloader_v21(self):
124128
Ported to the rawio API by Samuel Garcia.
125129
"""
126130

127-
dirname = self.get_filename_path('blackrock_2_1/l101210-001')
131+
dirname = self.get_local_path('blackrock/blackrock_2_1/l101210-001')
128132
# First run with parameters for ns5, then run with correct parameters for ns2
129-
parameters = [('blackrock_2_1/l101210-001_nev-02_ns5.mat',
133+
parameters = [('blackrock/blackrock_2_1/l101210-001_nev-02_ns5.mat',
130134
{'nsx_to_load': 5, 'nev_override': '-'.join([dirname, '02'])}, 96),
131-
('blackrock_2_1/l101210-001.mat', {'nsx_to_load': 2}, 6)]
135+
('blackrock/blackrock_2_1/l101210-001.mat', {'nsx_to_load': 2}, 6)]
132136
for param in parameters:
133137
# Load data from Matlab generated files
134-
ml = scipy.io.loadmat(self.get_filename_path(filename=param[0]))
138+
ml = scipy.io.loadmat(self.get_local_path(param[0]))
135139
lfp_ml = ml['lfp'] # (channel x time) LFP matrix
136140
ts_ml = ml['ts'] # spike time stamps
137141
elec_ml = ml['el'] # spike electrodes

0 commit comments

Comments
 (0)