Feat (Fetch workflow): Add new criteria to Allen Brain and improve docs (#127)

* Feat (Fetch workflow): Add new criteria to Allen Brain in improve docs

Author: adrien-berchet

docs/source/fetch_config.rst

@@ -0,0 +1,61 @@
+Fetch configuration file
+========================
+
+The ``Fetch`` workflow aims at downloading morphology files from different source databases:
+
+* `NeuroMorpho <https://neuromorpho.org>`_
+* `MouseLight <https://ml-neuronbrowser.janelia.org>`_
+* `Allen Brain <https://celltypes.brain-map.org>`_
+
+At the end of the ``Fetch`` workflow, the result folder will contain all the morphology files and
+a ``metadata.csv`` file which contains all the metadata of each morphology file.
+
+Each database needs a different configuration file. In each case, the configuration file is a JSON
+file containing a list of objects, each object being a configuration element that will be executed
+independently. Because of this, two identical configuration elements will fetch the same
+morphologies. The result folder will contain only one copy of each morphology file but the
+``metadata.csv`` file will contain duplicated entries, so it is possible to understand why the
+result folder does not contain the expected number of files and which filters should be updated.
+
+Each configuration element can contain the following entries:
+
+* ``nb_morphologies``: the number of morphologies to fetch using the filters of the current object.
+* ``seed``: the random seed used to choose which elements are fetched among the ones available using
+  the filters of the current object.
+* all other entries should be valid for the requested source API.
+
+This page shows examples of each type of configuration file.
+
+NeuroMorpho
+-----------
+
+The filter entries should any valid filter entry for the
+`NeuroMorpho API <https://neuromorpho.org/apiReference.html>`_.
+
+.. literalinclude:: ../../src/morphology_workflows/_templates/neuromorpho_config.json
+   :language: json
+
+MouseLight
+----------
+
+The filter entries support only the following entries:
+
+* ``brain_region``: the name of the brain region to filter.
+
+.. literalinclude:: ../../src/morphology_workflows/_templates/mouselight_config.json
+   :language: json
+
+Allen Brain
+-----------
+
+The filter entries support only the following entries:
+
+* ``species``: the name of the species to filter.
+* ``brain_region``: the name of the brain region to filter.
+* any other valid filter entry for the
+  `AllenSDK API <http://alleninstitute.github.io/AllenSDK/cell_types.html>`_ (the filter keys
+  should be chosen from the ones used to create
+  `these entries <https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/cell_types_api.py#L260>`_.
+
+.. literalinclude:: ../../src/morphology_workflows/_templates/allen_config.json
+   :language: json

docs/source/index.rst

@@ -10,6 +10,7 @@
    Home <self>
    cli
    luigi_cfg
+   fetch_config
    placeholders_config
    api_ref
    changelog

src/morphology_workflows/_templates/allen_config.json

@@ -2,20 +2,17 @@
     {
         "species": "Mus musculus",
         "brain_region": "VISli",
-        "cell_type": "interneuron",
         "nb_morphologies": 2,
         "seed": 0
     },
     {
         "species": "Mus musculus",
         "brain_region": "VISli",
-        "cell_type": "pyramidal",
         "seed": 0
     },
     {
         "species": "Mus musculus",
         "brain_region": "VISli",
-        "cell_type": "pyramidal",
         "nb_morphologies": 9999,
         "seed": 0
     }

src/morphology_workflows/tasks/fetch.py

@@ -166,18 +166,17 @@ def allen_download(self, config):
         api.allen_morphology_cache.mkdir(parents=True, exist_ok=True)
 
         for conf_element in config:
-            size = conf_element.get("nb_morphologies", float("inf"))
-            species = conf_element.get("species", None)
-            brain_region = conf_element.get("brain_region", None)
-            # cell_type = conf_element.get("cell_type", None)
+            criteria = copy.deepcopy(conf_element)
+            size = criteria.pop("nb_morphologies", float("inf"))
+            brain_region = criteria.pop("brain_region", None)
+            seed = criteria.pop("seed", None)
 
             mask = np.full(len(api.neurons), True, dtype=bool)
-            if species is not None:
-                mask = mask & (api.neurons.species == species)
             if brain_region is not None:
                 mask = mask & (api.neurons.structure_area_abbrev == brain_region)
-            # if cell_type is not None:
-            #     mask = mask & (api.neurons.structure_area_abbrev == region)
+
+            for key, value in criteria.items():
+                mask = mask & (api.neurons[key] == value)
 
             neurons = api.neurons.loc[mask]
 
@@ -187,9 +186,7 @@ def allen_download(self, config):
                 # Download some neurons
                 downloaded_neurons = self._neuron_paths(
                     api.download_neurons(
-                        neurons.sample(
-                            min(len(neurons), size), random_state=conf_element.get("seed", None)
-                        ).id.values,
+                        neurons.sample(min(len(neurons), size), random_state=seed).id.values,
                         load_neurons=False,
                     ),
                     api.allen_morphology_cache,

tests/data/allen_config_download.json

@@ -2,20 +2,18 @@
     {
         "species": "Mus musculus",
         "brain_region": "VISli",
-        "cell_type": "interneuron",
         "nb_morphologies": 2,
         "seed": 0
     },
     {
         "species": "Mus musculus",
         "brain_region": "VISli",
-        "cell_type": "pyramidal",
+        "dendrite_type": "spiny",
         "seed": 0
     },
     {
         "species": "Mus musculus",
         "brain_region": "VISli",
-        "cell_type": "pyramidal",
         "nb_morphologies": 9999,
         "seed": 0
     }

tests/test_download_morphs.py

@@ -132,23 +132,21 @@ def test_allen(prepare_dir, data_dir):
     # Check metadata
     expected = pd.DataFrame(
         {
-            "species": ["Mus musculus"] * 10,
-            "brain_region": ["VISli"] * 10,
-            "cell_type": ["interneuron"] * 2 + ["pyramidal"] * 8,
-            "nb_morphologies": [2] * 2 + [np.nan] * 4 + [9999] * 4,
-            "seed": [0] * 10,
+            "species": ["Mus musculus"] * 8,
+            "brain_region": ["VISli"] * 8,
+            "nb_morphologies": [2] * 2 + [np.nan] * 2 + [9999] * 4,
+            "seed": [0] * 8,
             "morphology": [
                 "555019563.swc",
                 "603402458.swc",
-                "555019563.swc",
                 "603402458.swc",
                 "526573598.swc",
-                "555241040.swc",
                 "555019563.swc",
                 "603402458.swc",
                 "526573598.swc",
                 "555241040.swc",
             ],
+            "dendrite_type": [np.nan] * 2 + ["spiny"] * 2 + [np.nan] * 4,
         }
     )