Merge pull request #984 from NeuralEnsemble/gh_actions_and_datalad

Gh actions and datalad

Author: JuliaSprenger

.circleci/config.yml

@@ -0,0 +1,65 @@
+name: neo.io and neo.rawio tests
+
+on:
+  pull_request:
+    branches: [master, gh_actions_and_datalad]
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  build-and-test:
+    name: Test on (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # "macos-latest", "windows-latest"
+        os: ["ubuntu-latest", ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          python-version: 3.8
+      - name: Get current year-month
+        id: date
+        run: echo "::set-output name=date::$(date+'%Y-%m')"
+      - uses: actions/cache@v2
+        # the cache for python package is reset:
+        #   * every month
+        #   * when requirements/requirements_testing change
+        id: cache-venv
+        with:
+          path: ~/test_env
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements_testing.txt') }}-${{ steps.date.outputs.date }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+      - name: Install dependencies
+        run: |
+          # this is for datalad and download testing datasets
+          sudo apt install git git-annex
+          # needed for correct operation of git/git-annex/DataLad
+          git config --global user.email "neo_ci@fake_mail.com"
+          git config --global user.name "neo CI"
+          # create an environment and install everything
+          python -m venv ~/test_env
+          source ~/test_env/bin/activate
+          python -m pip install --upgrade pip
+          # pip install -r requirements.txt
+          pip install -r requirements_testing.txt
+          pip install -e .
+      - name: Get ephy_testing_data current head hash
+        # the key depend on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
+        id: vars
+        run: |
+          echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)"
+      - uses: actions/cache@v2
+        id: cache-datasets
+        with:
+          path: ~/ephy_testing_data
+          key: ${{ runner.os }}-datasets-${{ steps.vars.outputs.HASH_EPHY_DATASET }}
+          restore-keys: |
+            ${{ runner.os }}-datasets
+      - name: Test with pytest
+        run: |
+          source ~/test_env/bin/activate
+          # only neo.rawio and neo.io
+          pytest --cov=neo neo/test/rawiotest
+          pytest --cov=neo neo/test/iotest

.github/workflows/full-test.yml

@@ -30,6 +30,7 @@ before_install:
   - pip install "numpy==$NUMPY_VERSION"
 install:
   - pip install -r requirements.txt
+  - pip install nose
   - pip install coveralls
   - pip install .
   - pip install pillow

.travis.yml

@@ -8,7 +8,6 @@
 logging_handler = logging.StreamHandler()
 
 from neo.core import *
-# ~ import neo.rawio
 from neo.io import *
 
 from neo.version import version as __version__

neo/__init__.py

@@ -198,8 +198,12 @@ def read_spiketrain(self, cluster_id, model,
             w = pq.Quantity(w, raw_data_units)
         else:
             w = None
+        if model.duration > 0.:
+            t_stop = model.duration
+        else:
+            t_stop = np.max(model.spike_times[idx])
         sptr = SpikeTrain(times=model.spike_times[idx],
-                          t_stop=model.duration, waveforms=w, units='s',
+                          t_stop=t_stop, waveforms=w, units='s',
                           sampling_rate=model.sample_rate * pq.Hz,
                           file_origin=self.filename,
                           **{'cluster_id': cluster_id})

neo/io/kwikio.py

@@ -76,6 +76,7 @@
     ('offset', 'float64'),
     ('stream_id', 'U64'),
 ]
+
 # TODO for later: add t_start and length in _signal_channel_dtype
 # this would simplify all t_start/t_stop stuff for each RawIO class
 

neo/rawio/baserawio.py

@@ -29,8 +29,11 @@
                             assert_sub_schema_is_lazy_loaded,
                             assert_children_empty)
 
-from neo.test.rawiotest.tools import (can_use_network, make_all_directories,
-                                   download_test_file, create_local_temp_dir)
+from neo.test.rawiotest.tools import can_use_network
+from neo.test.rawiotest.common_rawio_test import repo_for_test
+from neo.utils import (download_dataset,
+    get_local_testing_data_folder, HAVE_DATALAD)
+
 
 from neo.test.iotest.tools import (cleanup_test_file,
                                    close_object_safe, create_generic_io_object,
@@ -40,11 +43,8 @@
                                    iter_generic_readers, iter_read_objects,
                                    read_generic,
                                    write_generic)
-from neo.test.generate_datasets import generate_from_supported_objects
-
 
-# url_for_tests = "https://portal.g-node.org/neo/" #This is the old place
-url_for_tests = "https://web.gin.g-node.org/NeuralEnsemble/ephy_testing_data/raw/master/"
+from neo.test.generate_datasets import generate_from_supported_objects
 
 
 class BaseTestIO:
@@ -67,8 +67,8 @@ class BaseTestIO:
     # all IO test need to modify this:
     ioclass = None  # the IOclass to be tested
 
-    files_to_test = []  # list of files to test compliances
-    files_to_download = []  # when files are at G-Node
+    entities_to_test = []  # list of files to test compliances
+    entities_to_download = []  # when files are at gin
 
     # when reading then writing produces files with identical hashes
     hash_conserved_when_write_read = False
@@ -84,7 +84,6 @@ def setUp(self):
         '''
         Set up the test fixture.  This is run for every test
         '''
-        self.files_to_test = copy(self.__class__.files_to_test)
         self.higher = self.ioclass.supported_objects[0]
         self.shortname = self.ioclass.__name__.lower().rstrip('io')
         # these objects can both be written and read
@@ -93,11 +92,19 @@ def setUp(self):
         # these objects can be either written or read
         self.io_readorwrite = list(set(self.ioclass.readable_objects) |
                                    set(self.ioclass.writeable_objects))
-        self.create_local_dir_if_not_exists()
-        self.download_test_files_if_not_present()
-        self.files_generated = []
-        self.generate_files_for_io_able_to_write()
-        self.files_to_test.extend(self.files_generated)
+
+        if HAVE_DATALAD:
+            for remote_path in self.entities_to_download:
+                download_dataset(repo=repo_for_test, remote_path=remote_path)
+
+            self.files_generated = []
+            self.generate_files_for_io_able_to_write()
+
+            # be carefull self.entities_to_test is class attributes
+            self.files_to_test = [self.get_local_path(e) for e in self.entities_to_test]
+        else:
+            self.files_to_test = []
+            raise unittest.SkipTest("Requires datalad download of data from the web")
 
     def create_local_dir_if_not_exists(self):
         '''
@@ -169,11 +176,15 @@ def able_to_write_or_read(self, writeread=False, readwrite=False):
 
         return True
 
-    def get_filename_path(self, filename):
-        '''
-        Get the path to a filename in the current temporary file directory
-        '''
-        return os.path.join(self.local_test_dir, filename)
+    def get_local_base_folder(self):
+        return get_local_testing_data_folder()
+
+    def get_local_path(self, sub_path):
+        root_local_path = self.get_local_base_folder()
+        local_path = root_local_path / sub_path
+        # TODO later : remove the str when all IOs handle the pathlib.Path objects
+        local_path = str(local_path)
+        return local_path
 
     def generic_io_object(self, filename=None, return_path=False, clean=False):
         '''

neo/test/iotest/common_io_test.py

@@ -9,9 +9,13 @@
 
 
 class TestAlphaOmegaIO(BaseTestIO, unittest.TestCase):
-    files_to_test = ['File_AlphaOmega_1.map',
-                     'File_AlphaOmega_2.map']
-    files_to_download = files_to_test
+    entities_to_download =[
+        'alphaomega'
+    ]
+    entities_to_test = [
+        'alphaomega/File_AlphaOmega_1.map',
+        'alphaomega/File_AlphaOmega_2.map'
+    ]
     ioclass = AlphaOmegaIO
 
 

neo/test/iotest/test_alphaomegaio.py

@@ -16,11 +16,13 @@
 
 class TestAsciiSignalIOWithTestFiles(BaseTestIO, unittest.TestCase):
     ioclass = AsciiSignalIO
-    files_to_download = [  # 'File_asciisignal_1.asc',
-        'File_asciisignal_2.txt',
-        'File_asciisignal_3.txt',
+    entities_to_download = [
+        'asciisignal'
+    ]
+    entities_to_test = [
+        'asciisignal/File_asciisignal_2.txt',
+        'asciisignal/File_asciisignal_3.txt',
     ]
-    files_to_test = files_to_download
 
 
 class TestAsciiSignalIO(unittest.TestCase):

neo/test/iotest/test_asciisignalio.py

@@ -10,8 +10,12 @@
 
 class TestAsciiSpikeTrainIO(BaseTestIO, unittest.TestCase, ):
     ioclass = AsciiSpikeTrainIO
-    files_to_download = ['File_ascii_spiketrain_1.txt']
-    files_to_test = files_to_download
+    entities_to_download = [
+        'asciispiketrain'
+    ]
+    entities_to_test = [
+        'asciispiketrain/File_ascii_spiketrain_1.txt',
+    ]
 
 
 if __name__ == "__main__":