Merge pull request #431 from biothings/custom-uploader

Extend Manifest

Author: shuchenliu

biothings/hub/dataplugin/loaders/dumper.py.tpl

@@ -6,6 +6,7 @@ from config import DATA_ARCHIVE_ROOT
 
 from biothings.utils.common import uncompressall
 
+import $PLUGIN_MODULE
 
 import biothings.hub.dataload.dumper
 

biothings/hub/dataplugin/loaders/loader.py

@@ -271,6 +271,8 @@ def get_dumper_dynamic_class(
                 dumper_class = self.dumper_registry.get(scheme)
                 dumper_configuration["BASE_CLASSES"] = "biothings.hub.dataload.dumper.%s" % dumper_class.__name__
 
+            dumper_configuration["PLUGIN_MODULE"] = dumper_configuration["BASE_CLASSES"].split(".")[0]
+
             if not dumper_class:
                 raise LoaderException("No dumper class registered to handle scheme '%s'", scheme)
 
@@ -438,7 +440,16 @@ def jobs(self):
                         )
                     )
                 else:
-                    confdict["BASE_CLASSES"] = "biothings.hub.dataload.uploader.BaseSourceUploader"
+                    # use specified custom class
+                    klass = uploader_section.get("class")
+                    if klass:
+                        get_class_from_classpath(klass)
+                        confdict["BASE_CLASSES"] = klass
+                    else:
+                        confdict["BASE_CLASSES"] = "biothings.hub.dataload.uploader.BaseSourceUploader"
+
+                    confdict["PLUGIN_MODULE"] = confdict["BASE_CLASSES"].split(".")[0]
+
                     confdict["JOBS_FUNC"] = ""
 
                 if uploader_section.get("mapping"):

biothings/hub/dataplugin/loaders/schema/manifest.json

@@ -143,6 +143,10 @@
                 "disabled": {
                     "type": "boolean",
                     "description": "If true, then the dumper will not be run. This is useful for testing purposes or if you want to disable the dumper without removing it from the manifest"
+                },
+                "class": {
+                  "type": "string",
+                  "description": "Reference to a locally defined dumper class. Format: 'module:class_name'"
                 }
             },
             "required": [
@@ -264,7 +268,11 @@
                         "examples": [
                             "parallelizer:parallel_jobs"
                         ]
-                    }
+                    },
+                    "class": {
+                      "type": "string",
+                      "description": "Reference to a locally defined uploader class. Format: 'module:class_name'"
+                  }
                 },
                 "required": [
                     "name",

biothings/hub/dataplugin/loaders/subuploader.py.tpl

@@ -4,7 +4,7 @@ import biothings, config
 biothings.config_for_app(config)
 
 import biothings.hub.dataload.uploader
-
+import $PLUGIN_MODULE
 
 $PARSER_FACTORY_CODE
 

biothings/hub/dataplugin/loaders/uploader.py.tpl

@@ -5,6 +5,7 @@ biothings.config_for_app(config)
 
 import biothings.hub.dataload.uploader
 
+import $PLUGIN_MODULE
 
 $PARSER_FACTORY_CODE