online tests for all datasets

- add tests to test_datasets
- fixed error in vesicles dataset
- normalized description files
- updated CHANGELOG

Author: orbeckst

CHANGELOG.md

@@ -5,9 +5,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixes
+- vesicles dataset: failed to get description
+
 ### Added
 - progressbar for downloads (#29)
 
+### Changes (internal)
+- tests with full downloads can be performed with 
+  `pytest -m online`; by default `pytest -m 'not online'` is run,
+  which skips downloading gigabytes of data (PR #18)
+  
 
 ## [0.4.0] - 2018-10-05
 ### Added

MDAnalysisData/descr/adk_transitions_DIMS.rst

@@ -1,4 +1,5 @@
 .. -*- coding: utf-8 -*-
+
 .. _`adk-transitions-DIMS-dataset`:
 
 AdK DIMS transitions ensemble dataset

MDAnalysisData/descr/adk_transitions_FRODA.rst

@@ -1,4 +1,5 @@
 .. -*- coding: utf-8 -*-
+
 .. _`adk-transitions-FRODA-dataset`:
 
 

MDAnalysisData/tests/test_base.py

@@ -118,14 +118,13 @@ def test_fetch_remote_sha_success(remote_topology, mocker, dirname):
     exp = os.path.join(dirname, remote_topology.filename)
     assert base._fetch_remote(remote_topology, dirname=dirname) == exp
 
-def test_lazy_fetch(remote_topology, tmpdir, mocker):
+def test_lazy_fetch(tmpdir, mocker):
     mocker.patch('MDAnalysisData.adk_equilibrium.exists', return_value=True)
     fr = mocker.patch('MDAnalysisData.adk_equilibrium._fetch_remote')
     # check the laziness of grabbing a dataset
     # - mock exists to always say true
     # - grab the "dataset" then check no remote calls were done
     stuff = adk_equilibrium.fetch_adk_equilibrium()
-
     assert not fr.called
 
 

MDAnalysisData/tests/test_datasets.py

@@ -11,14 +11,18 @@
 from __future__ import absolute_import, division
 
 import os.path
+import glob
 
 import pytest
 
 from MDAnalysisData import base
 from MDAnalysisData import datasets
 from MDAnalysisData import adk_equilibrium
 from MDAnalysisData import ifabp_water
-
+from MDAnalysisData import nhaa_equilibrium
+from MDAnalysisData import CG_fiber
+from MDAnalysisData import vesicles
+from MDAnalysisData import adk_transitions
 
 # For filetype=topology, the data are downloaded and cached.
 # For filetype=trajectory the cached data are used.
@@ -38,7 +42,9 @@ def test_adk_equilibrium(filetype):
     assert os.path.exists(data[filetype])
 
 @pytest.mark.online
-@pytest.mark.parametrize('filetype', ('topology', 'trajectory'))
+@pytest.mark.parametrize('filetype', ('topology',
+                                      'trajectory',
+                                      'structure'))
 def test_ifabp_water(filetype):
     data = datasets.fetch_ifabp_water()
 
@@ -50,3 +56,77 @@ def test_ifabp_water(filetype):
     assert os.path.basename(data[filetype]) == metadata[filetype].filename
     assert os.path.exists(data[filetype])
 
+
+@pytest.mark.online
+@pytest.mark.parametrize('filetype', ('topology', 'trajectory'))
+def test_nhaa_equilibrium(filetype):
+    data = datasets.fetch_nhaa_equilibrium()
+
+    metadata = nhaa_equilibrium.ARCHIVE
+
+    assert len(data.DESCR) == 1475
+    assert data.DESCR.startswith(".. -*- coding: utf-8 -*-\n\n.. _`nhaa-equilibrium-dataset`:")
+
+    assert os.path.basename(data[filetype]) == metadata[filetype].filename
+    assert os.path.exists(data[filetype])
+
+
+@pytest.mark.online
+@pytest.mark.parametrize('filetype', ('topology', 'trajectory'))
+def test_CG_fiber(filetype):
+    data = datasets.fetch_CG_fiber()
+
+    metadata = CG_fiber.ARCHIVE
+
+    assert len(data.DESCR) == 535
+    assert data.DESCR.startswith(".. -*- coding: utf-8 -*-\n\n.. _`CG_fiber-dataset`:")
+
+    assert os.path.basename(data[filetype]) == metadata[filetype].filename
+    assert os.path.exists(data[filetype])
+
+
+@pytest.mark.online
+@pytest.mark.parametrize('filename',
+                         vesicles.METADATA['vesicle_lib']['CONTENTS']['structures'])
+def test_vesicles(filename):
+    data = datasets.fetch_vesicle_lib()
+
+    metadata = vesicles.METADATA['vesicle_lib']
+
+    assert len(data.DESCR) == 1329
+    assert data.DESCR.startswith(".. -*- coding: utf-8 -*-\n\n.. _`vesicle-library-dataset`:")
+
+    assert data.N_structures == metadata['CONTENTS']['N_structures']
+    assert data.N_structures == len(metadata['CONTENTS']['structures'])
+    assert any(path.endswith(filename) for path in data.structures)
+    assert any(os.path.exists(path) for path in data.structures if path.endswith(filename))
+
+
+@pytest.mark.online
+@pytest.mark.parametrize('method,descr_length',
+                         (('DIMS', 1395),
+                          ('FRODA', 1399)))
+@pytest.mark.parametrize('filetype', ('topology', 'trajectories'))
+def test_adk_transitions(method, descr_length, filetype):
+    if method == "DIMS":
+        data = datasets.fetch_adk_transitions_DIMS()
+    elif method == "FRODA":
+        data = datasets.fetch_adk_transitions_FRODA()
+    else:
+        raise ValueError("Unknown adk_transitions method '{}'".format(method))
+
+    metadata = adk_transitions.METADATA[method]['CONTENTS']
+
+    assert len(data.DESCR) == descr_length
+    descr_header = (".. -*- coding: utf-8 -*-\n\n"
+                    ".. _`adk-transitions-{}-dataset`:".format(
+                        method))
+    assert data.DESCR.startswith(descr_header)
+
+    datafiles = data[filetype] if isinstance(data[filetype], list) \
+        else [data[filetype]]
+    pattern = os.path.join("*", metadata[filetype])
+
+    assert all(glob.fnmatch.fnmatch(path, pattern) for path in datafiles), \
+        "not all datafiles {} match {}".format(datafiles, pattern)
+    assert all(os.path.exists(path) for path in datafiles)

MDAnalysisData/vesicles.py

@@ -106,6 +106,6 @@ def fetch_vesicle_lib(data_home=None, download_if_missing=True):
                                len(records.structures),
                                records.N_structures))
 
-    records.DESCR = _read_description(DESCRIPTION)
+    records.DESCR = _read_description(metadata['DESCRIPTION'])
 
     return records