Add tests to evaluate the new metatypes

Author: ctrl-schaff

tests/hub/dataplugin/conftest.py

@@ -1,7 +1,10 @@
 """
-Fixtures for mocking different types of plugin types
+Fixtures for mocking different types of plugin types,
+along with miscellaneous infrastructure
 """
 
+import asyncio
+
 from pathlib import Path
 import functools
 import http.server
@@ -15,20 +18,19 @@
 
 import pytest
 
-
 logger = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope="session")
 def temporary_mock_data(tmp_path_factory):
     """
-    Generates a subset of random binary files for populating the mock data hosting
+    Generates binary files for populating the mock data hosting
     """
     set_file_size_bytes = [1024, 2048, 4096, 8192]
 
     temp_directory_name = "submarine"
     temp_directory = tmp_path_factory.mktemp(temp_directory_name)
-    num_data_files = random.randint(3, 7)
+    num_data_files = 15
 
     for file_index, file_size_bytes in enumerate(random.choices(set_file_size_bytes, k=num_data_files)):
         random_binary_filename = f"dleaf{file_index}"

tests/hub/dataplugin/data/plugin_architectures/basic_dumper_plugin/manifest.json

@@ -0,0 +1,12 @@
+{
+    "version": "1.0",
+    "dumper": {
+        "data_url": [],
+        "uncompress": false,
+        "release": "version:get_release"
+    },
+    "uploader": {
+        "parser": "parser:size_loader",
+        "on_duplicates": "error"
+    }
+}

tests/hub/dataplugin/data/plugin_architectures/basic_dumper_plugin/mapping.py

@@ -0,0 +1,8 @@
+def size_mapping(cls):
+    """
+    Mock elasticsearch mapping for the size loader
+    """
+    elasticsearch_mapping = {
+        "associatedWith": {"properties": {"name": {"type": "keyword"}, "size": {"type": "keyword"}}}
+    }
+    return elasticsearch_mapping

tests/hub/dataplugin/data/plugin_architectures/basic_dumper_plugin/parser.py

@@ -0,0 +1,23 @@
+"""
+Mock handler for parsing data associated with our plugin
+"""
+
+from pathlib import Path
+import uuid
+
+
+def size_loader(data_folder):
+    """
+    Iterates over the data folder and generates documents of the following structure:
+    {
+        "name": <filename>
+        "size": <filesize>
+    }
+    """
+    for file_path_object in Path(data_folder).glob("**/*"):
+        document = {
+            "_id": str(uuid.uuid4()),
+            "name": file_path_object.name,
+            "size": Path(file_path_object).stat().st_size,
+        }
+        yield document

tests/hub/dataplugin/data/plugin_architectures/basic_dumper_plugin/version.py

@@ -0,0 +1,3 @@
+def get_release(self):
+    release_date = "1970-01-01"
+    return release_date

tests/hub/dataplugin/data/plugin_architectures/disabled_dumper_plugin/manifest.json

@@ -0,0 +1,13 @@
+{
+    "version": "1.0",
+    "dumper": {
+        "data_url": [],
+        "uncompress": false,
+        "release": "version:get_release",
+        "disabled": true
+    },
+    "uploader": {
+        "parser": "parser:size_loader",
+        "on_duplicates": "error"
+    }
+}

tests/hub/dataplugin/data/plugin_architectures/disabled_dumper_plugin/mapping.py

@@ -0,0 +1,8 @@
+def size_mapping(cls):
+    """
+    Mock elasticsearch mapping for the size loader
+    """
+    elasticsearch_mapping = {
+        "associatedWith": {"properties": {"name": {"type": "keyword"}, "size": {"type": "keyword"}}}
+    }
+    return elasticsearch_mapping

tests/hub/dataplugin/data/plugin_architectures/disabled_dumper_plugin/parser.py

@@ -0,0 +1,23 @@
+"""
+Mock handler for parsing data associated with our plugin
+"""
+
+from pathlib import Path
+import uuid
+
+
+def size_loader(data_folder):
+    """
+    Iterates over the data folder and generates documents of the following structure:
+    {
+        "name": <filename>
+        "size": <filesize>
+    }
+    """
+    for file_path_object in Path(data_folder).glob("**/*"):
+        document = {
+            "_id": str(uuid.uuid4()),
+            "name": file_path_object.name,
+            "size": Path(file_path_object).stat().st_size,
+        }
+        yield document

tests/hub/dataplugin/data/plugin_architectures/disabled_dumper_plugin/version.py

@@ -0,0 +1,3 @@
+def get_release(self):
+    release_date = "1970-01-01"
+    return release_date

tests/hub/dataplugin/test_dump_job.py

@@ -0,0 +1,70 @@
+"""
+Integration test for evaluating the various pieces
+required to execute a dump job within our hub
+"""
+
+import asyncio
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from biothings import config
+from biothings.hub.dataload.dumper import DumperManager
+from biothings.utils.manager import JobManager
+from biothings.hub.dataload.uploader import UploaderManager
+from biothings.hub.dataplugin.assistant import LocalAssistant
+from biothings.hub.dataplugin.manager import DataPluginManager
+from biothings.utils import hub_db
+
+plugin_designs = ["single_uploader_plugin", "multiple_uploader_plugin"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("plugin", plugin_designs, indirect=True)
+async def test_job_dump_operation(plugin):
+    hub_db.setup(config)
+
+    # construct our manager instances
+    job_manager = JobManager(
+        loop=asyncio.get_running_loop(),
+        process_queue=None,
+        thread_queue=None,
+        max_memory_usage=None,
+        num_workers=None,
+        num_threads=None,
+        auto_recycle=True,
+    )
+
+    LocalAssistant.data_plugin_manager = DataPluginManager(job_manager=None)
+    LocalAssistant.uploader_manager = UploaderManager(job_manager=None)
+    LocalAssistant.dumper_manager = DumperManager(job_manager=job_manager, poll_schedule=None, datasource_path=None)
+
+    plugin_name = plugin.name
+    assistant_url = f"local://{plugin_name}"
+    assistant_instance = LocalAssistant(assistant_url)
+
+    data_plugin_entry = hub_db.get_data_plugin()
+    data_plugin_entry.remove({"_id": assistant_instance.plugin_name})
+    plugin_entry = {
+        "_id": assistant_instance.plugin_name,
+        "plugin": {
+            "url": assistant_url,
+            "type": assistant_instance.plugin_type,
+            "active": True,
+        },
+        "download": {"data_folder": str(Path(plugin))},
+    }
+
+    data_plugin_entry.insert_one(plugin_entry)
+
+    plugin_loader = assistant_instance.loader
+    plugin_loader.load_plugin()
+
+    current_plugin = SimpleNamespace(
+        plugin_name=plugin_name,
+        data_plugin_dir=plugin,
+        in_plugin_dir=plugin_name is None,
+    )
+
+    await assistant_instance.dumper_manager.dump_all()