Merge remote-tracking branch 'origin/50-clowder20-submit-file-to-extractor' into 50-clowder20-submit-file-to-extractor

Author: tcnichol

pyclowder/api/v1/datasets.py

@@ -10,10 +10,6 @@
 from pyclowder.collections import get_datasets, get_child_collections, delete as delete_collection
 from pyclowder.utils import StatusMessage
 
-from dotenv import load_dotenv
-load_dotenv()
-clowder_version = float(os.getenv('clowder_version'))
-
 
 def create_empty(connector, host, key, datasetname, description, parentid=None, spaceid=None):
     """Create a new dataset in Clowder.

pyclowder/api/v2/files.py

@@ -15,10 +15,6 @@
 from pyclowder.datasets import get_file_list
 from pyclowder.collections import get_datasets, get_child_collections
 
-from dotenv import load_dotenv
-load_dotenv()
-clowder_version = float(os.getenv('clowder_version'))
-
 # Some sources of urllib3 support warning suppression, but not all
 try:
     from urllib3 import disable_warnings

pyclowder/connectors.py

@@ -55,11 +55,6 @@
 from email.mime.multipart import MIMEMultipart
 from string import Template
 
-from dotenv import load_dotenv
-load_dotenv()
-
-clowder_version = float(os.getenv('clowder_version', '1.0'))
-
 
 class Connector(object):
     """ Class that will listen for messages.
@@ -139,7 +134,7 @@ def alive(self):
         """Return whether connection is still alive or not."""
         return True
 
-    def _build_resource(self, body, host, secret_key):
+    def _build_resource(self, body, host, secret_key, clowder_version):
         """Examine message body and create resource object based on message type.
 
         Example FILE message -- *.file.#
@@ -242,7 +237,7 @@ def _build_resource(self, body, host, secret_key):
 
         elif resource_type == "file":
             ext = os.path.splitext(filename)[1]
-            if float(os.getenv('clowder_version')) == 2.0:
+            if clowder_version == 2:
                 return {
                     "type": "file",
                     "id": fileid,
@@ -407,22 +402,18 @@ def _process_message(self, body):
         if not host.endswith('/'): host += '/'
         secret_key = body.get('secretKey', '')
         retry_count = 0 if 'retry_count' not in body else body['retry_count']
-        resource = self._build_resource(body, host, secret_key)
+        clowder_version = int(body.get('clowderVersion', os.getenv('CLOWDER_VERSION', '1')))
+        resource = self._build_resource(body, host, secret_key, clowder_version)
         if not resource:
             logging.error("No resource found, this is bad.")
             return
 
         # register extractor
-        if clowder_version >= 2.0:
-            url = "%sapi/v2/extractors" % source_host
-        else:
+        if clowder_version != 2:
             url = "%sapi/extractors" % source_host
-        if url not in Connector.registered_clowder:
-            Connector.registered_clowder.append(url)
-            if clowder_version >= 2.0:
+            if url not in Connector.registered_clowder:
+                Connector.registered_clowder.append(url)
                 self.register_extractor("%s?key=%s" % (url,secret_key))
-            else:
-                self.register_extractor("%s?key=%s" % (url, secret_key))
 
         # tell everybody we are starting to process the file
         self.status_update(pyclowder.utils.StatusMessage.start, resource, "Started processing.")
@@ -442,18 +433,10 @@ def _process_message(self, body):
                         found_local = False
                         try:
                             if check_result != pyclowder.utils.CheckMessage.bypass:
-                                if clowder_version >= 2.0:
-                                    file_metadata = pyclowder.files.download_info(self, host, secret_key, resource["id"])
-                                else:
-                                    file_metadata = pyclowder.files.download_info(self, host, secret_key, resource["id"])
+                                file_metadata = pyclowder.files.download_info(self, host, secret_key, resource["id"])
                                 file_path = self._check_for_local_file(file_metadata)
                                 if not file_path:
-                                    if clowder_version >= 2.0:
-                                        file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
-                                                                         resource["intermediate_id"],
-                                                                         resource["file_ext"])
-                                    else:
-                                        file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
+                                    file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
                                                                              resource["intermediate_id"],
                                                                              resource["file_ext"])
                                 else:
@@ -539,48 +522,24 @@ def register_extractor(self, endpoints):
         This assumes a file called extractor_info.json to be located in either the
         current working directory, or the folder where the main program is started.
         """
-        if clowder_version >= 2.0:
-            if not endpoints or endpoints == "":
-                return
-
-            logger = logging.getLogger(__name__)
-
-            headers = {'Content-Type': 'application/json'}
-            data = self.extractor_info
-
-            for url in endpoints.split(','):
-                if url not in Connector.registered_clowder:
-                    Connector.registered_clowder.append(url)
-                    try:
-                        result = requests.post(url.strip(), headers=headers,
-                                               data=json.dumps(data),
-                                               verify=self.ssl_verify)
-                        result.raise_for_status()
-                        logger.debug("Registering extractor with %s : %s", url, result.text)
-                    except Exception as exc:  # pylint: disable=broad-except
-                        logger.exception('Error in registering extractor: ' + str(exc))
-        else:
-            # don't do any work if we wont register the endpoint
-            if not endpoints or endpoints == "":
-                return
-
-            logger = logging.getLogger(__name__)
-
-            headers = {'Content-Type': 'application/json'}
-            data = self.extractor_info
+        if not endpoints or endpoints == "":
+            return
 
+        logger = logging.getLogger(__name__)
 
+        headers = {'Content-Type': 'application/json'}
+        data = self.extractor_info
 
-            for url in endpoints.split(','):
-                if url not in Connector.registered_clowder:
-                    Connector.registered_clowder.append(url)
-                    try:
-                        result = requests.post(url.strip(), headers=headers,
-                                               data=json.dumps(data),
-                                               verify=self.ssl_verify)
-                        result.raise_for_status()
-                        logger.debug("Registering extractor with %s : %s", url, result.text)
-                    except Exception as exc:  # pylint: disable=broad-except
+        for url in endpoints.split(','):
+            if url not in Connector.registered_clowder:
+                Connector.registered_clowder.append(url)
+                try:
+                    result = requests.post(url.strip(), headers=headers,
+                                           data=json.dumps(data),
+                                           verify=self.ssl_verify)
+                    result.raise_for_status()
+                    logger.debug("Registering extractor with %s : %s", url, result.text)
+                except Exception as exc:  # pylint: disable=broad-except
                         logger.exception('Error in registering extractor: ' + str(exc))
 
     # pylint: disable=no-self-use
@@ -742,7 +701,7 @@ def connect(self):
 
             self.channel.queue_bind(queue=self.rabbitmq_queue,
                                     exchange=self.rabbitmq_exchange,
-                                    routing_key="extractors." + self.extractor_name)
+                                    routing_key=self.extractor_name)
 
         # start the extractor announcer
         self.announcer = RabbitMQBroadcast(self.rabbitmq_uri, self.extractor_info, self.rabbitmq_queue, self.heartbeat)

pyclowder/datasets.py

@@ -15,10 +15,7 @@
 from pyclowder.collections import get_datasets, get_child_collections, delete as delete_collection
 from pyclowder.utils import StatusMessage
 
-from dotenv import load_dotenv
-load_dotenv()
-clowder_version = float(os.getenv('clowder_version', '1.0'))
-
+clowder_version = int(os.getenv('CLOWDER_VERSION', '1'))
 
 def create_empty(connector, host, key, datasetname, description, parentid=None, spaceid=None):
     """Create a new dataset in Clowder.
@@ -32,7 +29,7 @@ def create_empty(connector, host, key, datasetname, description, parentid=None,
     parentid -- id of parent collection
     spaceid -- id of the space to add dataset to
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         datasetid = v2datasets.create_empty(connector, host, key, datasetname, description, parentid, spaceid)
     else:
         datasetid = v1datasets.create_empty(connector, host, key, datasetname, description, parentid, spaceid)
@@ -48,7 +45,7 @@ def delete(connector, host, key, datasetid):
     key -- the secret key to login to clowder
     datasetid -- the dataset to delete
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         result = v2datasets.delete(connector, host, key, datasetid)
     else:
         result = v2datasets.delete(connector, host, key, datasetid)
@@ -90,7 +87,7 @@ def download(connector, host, key, datasetid):
     key -- the secret key to login to clowder
     datasetid -- the file that is currently being processed
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         zipfile = v2datasets.download(connector, host, key, datasetid)
     else:
         zipfile = v1datasets.download(connector, host, key, datasetid)
@@ -107,7 +104,7 @@ def download_metadata(connector, host, key, datasetid, extractor=None):
     datasetid -- the dataset to fetch metadata of
     extractor -- extractor name to filter results (if only one extractor's metadata is desired)
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         result_json = v2datasets.download_metadata(connector, host, key, datasetid, extractor)
         return result_json
     else:
@@ -124,7 +121,7 @@ def get_info(connector, host, key, datasetid):
     key -- the secret key to login to clowder
     datasetid -- the dataset to get info of
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         info = v2datasets.get_info(connector, host, key, datasetid)
     else:
         info = v1datasets.get_info(connector, host, key, datasetid)
@@ -140,7 +137,7 @@ def get_file_list(connector, host, key, datasetid):
     key -- the secret key to login to clowder
     datasetid -- the dataset to get filelist of
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         file_list = v2datasets.get_file_list(connector, host, key, datasetid)
     else:
         file_list = v1datasets.get_file_list(connector, host, key, datasetid)
@@ -158,7 +155,7 @@ def remove_metadata(connector, host, key, datasetid, extractor=None):
     extractor -- extractor name to filter deletion
                     !!! ALL JSON-LD METADATA WILL BE REMOVED IF NO extractor PROVIDED !!!
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         v2datasets.remove_metadata(connector, host, key, datasetid, extractor)
     else:
         v1datasets.remove_metadata(connector, host, key, datasetid, extractor)
@@ -174,7 +171,7 @@ def submit_extraction(connector, host, key, datasetid, extractorname):
     datasetid -- the dataset UUID to submit
     extractorname -- registered name of extractor to trigger
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         result_status_code = v2datasets.submit_extraction(connector, host, key, datasetid, extractorname)
     else:
         result_status_code = v1datasets.submit_extraction(connector, host, key, datasetid, extractorname)
@@ -235,7 +232,7 @@ def upload_metadata(connector, host, key, datasetid, metadata):
     datasetid -- the dataset that is currently being processed
     metadata -- the metadata to be uploaded
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         v2datasets.upload_metadata(connector, host, key, datasetid, metadata)
     else:
         v1datasets.upload_metadata(connector, host, key, datasetid, metadata)

pyclowder/extractors.py

@@ -24,9 +24,7 @@
 import pyclowder.files
 import pyclowder.datasets
 
-from dotenv import load_dotenv
-load_dotenv()
-clowder_version = float(os.getenv('clowder_version', '1.0'))
+clowder_version = int(os.getenv('CLOWDER_VERSION', '1'))
 
 
 class Extractor(object):
@@ -266,7 +264,7 @@ def get_metadata(self, content, resource_type, resource_id, server=None):
                 if not self._check_key(k, self.extractor_info['contexts']):
                     logger.debug("Simple check could not find %s in contexts" % k)
         # TODO generate clowder2.0 extractor info
-        if clowder_version >= 2.0:
+        if clowder_version == 2:
             new_extractor_info = self._get_extractor_info_v2()
             md = dict()
             md["file_version"] = 1

pyclowder/files.py

@@ -16,9 +16,8 @@
 from pyclowder.collections import get_datasets, get_child_collections
 import pyclowder.api.v2.files as v2files
 import pyclowder.api.v1.files as v1files
-from dotenv import load_dotenv
-load_dotenv()
-clowder_version = float(os.getenv('clowder_version', '1.0'))
+
+clowder_version = int(os.getenv('CLOWDER_VERSION', '1'))
 
 # Some sources of urllib3 support warning suppression, but not all
 try:
@@ -41,7 +40,7 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
     intermediatefileid -- either same as fileid, or the intermediate file to be used
     ext -- the file extension, the downloaded file will end with this extension
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         inputfilename = v2files.download(connector, host, key, fileid, intermediatefileid, ext)
     else:
         inputfilename = v1files.download(connector, host, key, fileid, intermediatefileid, ext)
@@ -58,7 +57,7 @@ def download_info(connector, host, key, fileid):
     fileid -- the file to fetch metadata of
     """
 
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         result = v2files.download_info(connector, host, key, fileid)
     else:
         result = v1files.download_info(connector, host, key, fileid)
@@ -75,7 +74,7 @@ def download_metadata(connector, host, key, fileid, extractor=None):
     fileid -- the file to fetch metadata of
     extractor -- extractor name to filter results (if only one extractor's metadata is desired)
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         result = v2files.download_metadata(connector, host, key, fileid, extractor)
     else:
         result = v1files.download_metadata(connector, host, key, fileid, extractor)
@@ -92,7 +91,7 @@ def submit_extraction(connector, host, key, fileid, extractorname):
     fileid -- the file UUID to submit
     extractorname -- registered name of extractor to trigger
     """
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         result = v2files.submit_extraction(connector, host, key, fileid, extractorname)
     else:
         result = v1files.submit_extraction(connector, host, key, fileid, extractorname)
@@ -162,7 +161,7 @@ def upload_metadata(connector, host, key, fileid, metadata):
     metadata -- the metadata to be uploaded
     """
 
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         v2files.upload_metadata(connector, host, key, fileid, metadata)
     else:
         v1files.upload_metadata(connector, host, key, fileid, metadata)
@@ -277,7 +276,7 @@ def upload_to_dataset(connector, host, key, datasetid, filepath, check_duplicate
     check_duplicate -- check if filename already exists in dataset and skip upload if so
     """
 
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         v2files.upload_to_dataset(connector, host, key, datasetid, filepath, check_duplicate)
     else:
         logger = logging.getLogger(__name__)
@@ -322,7 +321,7 @@ def _upload_to_dataset_local(connector, host, key, datasetid, filepath):
     filepath -- path to file
     """
 
-    if clowder_version >= 2.0:
+    if clowder_version == 2:
         uploadedfileid = v2files._upload_to_dataset_local(connector, host, key, datasetid, filepath)
     else:
         uploadedfileid = v1files._upload_to_dataset_local(connector, host, key, datasetid, filepath)

requirements.txt

@@ -22,4 +22,3 @@ requests-toolbelt==0.9.1
     # via pyclowder (setup.py)
 urllib3==1.26.8
     # via requests
-python-dotenv