adding new methods for making pyclowder compatible with clowder v2

at different points in the code, the version will be checked
different methods will use different endpoints and will use the bearer token instead of the key

Author: tcnichol

pyclowder/connectors.py

@@ -55,6 +55,9 @@
 from email.mime.multipart import MIMEMultipart
 from string import Template
 
+from dotenv import load_dotenv
+load_dotenv()
+
 
 class Connector(object):
     """ Class that will listen for messages.
@@ -179,6 +182,8 @@ def _build_resource(self, body, host, secret_key):
         intermediatefileid = body.get('intermediateId', '')
         datasetid = body.get('datasetId', '')
         filename = body.get('filename', '')
+        if float(os.getenv('clowder_version')) == 2.0:
+            token = body.get('token', ' ')
 
         # determine resource type; defaults to file
         resource_type = "file"
@@ -237,15 +242,27 @@ def _build_resource(self, body, host, secret_key):
 
         elif resource_type == "file":
             ext = os.path.splitext(filename)[1]
-            return {
-                "type": "file",
-                "id": fileid,
-                "intermediate_id": intermediatefileid,
-                "name": filename,
-                "file_ext": ext,
-                "parent": {"type": "dataset",
-                           "id": datasetid}
-            }
+            if float(os.getenv('clowder_version')) == 2.0:
+                return {
+                    "type": "file",
+                    "id": fileid,
+                    "intermediate_id": intermediatefileid,
+                    "name": filename,
+                    "file_ext": ext,
+                    "token": token,
+                    "parent": {"type": "dataset",
+                               "id": datasetid}
+                }
+            else:
+                return {
+                    "type": "file",
+                    "id": fileid,
+                    "intermediate_id": intermediatefileid,
+                    "name": filename,
+                    "file_ext": ext,
+                    "parent": {"type": "dataset",
+                               "id": datasetid}
+                }
 
         elif resource_type == "metadata":
             return {
@@ -390,17 +407,26 @@ def _process_message(self, body):
         if not source_host.endswith('/'): source_host += '/'
         if not host.endswith('/'): host += '/'
         secret_key = body.get('secretKey', '')
+        token = body.get('token', ' ')
         retry_count = 0 if 'retry_count' not in body else body['retry_count']
         resource = self._build_resource(body, host, secret_key)
         if not resource:
             logging.error("No resource found, this is bad.")
             return
 
         # register extractor
-        url = "%sapi/extractors" % source_host
-        if url not in Connector.registered_clowder:
-            Connector.registered_clowder.append(url)
-            self.register_extractor("%s?key=%s" % (url, secret_key))
+        # TODO make work for clowder2.0
+        if float(os.getenv('clowder_version')) == 2.0:
+            print('do differently')
+            registration_url  = "%sapi/v2/extractors" % source_host
+            if registration_url not in Connector.registered_clowder:
+                Connector.registered_clowder.append(registration_url)
+                self.register_extractor_v2(registration_url, token)
+        else:
+            url = "%sapi/extractors" % source_host
+            if url not in Connector.registered_clowder:
+                Connector.registered_clowder.append(url)
+                self.register_extractor("%s?key=%s" % (url, secret_key))
 
         # tell everybody we are starting to process the file
         self.status_update(pyclowder.utils.StatusMessage.start, resource, "Started processing.")
@@ -420,12 +446,20 @@ def _process_message(self, body):
                         found_local = False
                         try:
                             if check_result != pyclowder.utils.CheckMessage.bypass:
-                                file_metadata = pyclowder.files.download_info(self, host, secret_key, resource["id"])
+                                if float(os.getenv('clowder_version')) == 2.0:
+                                    file_metadata = pyclowder.files.download_info_v2(self, host, token, resource["id"])
+                                else:
+                                    file_metadata = pyclowder.files.download_info(self, host, secret_key, resource["id"])
                                 file_path = self._check_for_local_file(file_metadata)
                                 if not file_path:
-                                    file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
-                                                                         resource["intermediate_id"],
-                                                                         resource["file_ext"])
+                                    if float(os.getenv('clowder_version')) == 2.0:
+                                        file_path = pyclowder.files.download_v2(self, host, token, resource["id"],
+                                                                             resource["intermediate_id"],
+                                                                             resource["file_ext"])
+                                    else:
+                                        file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
+                                                                             resource["intermediate_id"],
+                                                                             resource["file_ext"])
                                 else:
                                     found_local = True
                                 resource['local_paths'] = [file_path]
@@ -531,6 +565,34 @@ def register_extractor(self, endpoints):
                 except Exception as exc:  # pylint: disable=broad-except
                     logger.exception('Error in registering extractor: ' + str(exc))
 
+    def register_extractor_v2(self, endpoint, token):
+        """Register extractor info with Clowder.
+
+        This assumes a file called extractor_info.json to be located in either the
+        current working directory, or the folder where the main program is started.
+        """
+
+        # don't do any work if we wont register the endpoint
+        if not endpoint or endpoint == "":
+            return
+
+        logger = logging.getLogger(__name__)
+
+        headers = {'Content-Type': 'application/json',
+                   'Authorization': 'Bearer ' + token}
+        data = self.extractor_info
+
+        if endpoint not in Connector.registered_clowder:
+            Connector.registered_clowder.append(endpoint)
+            try:
+                result = requests.post(endpoint.strip(), headers=headers,
+                                       data=json.dumps(data),
+                                       verify=self.ssl_verify)
+                result.raise_for_status()
+                logger.debug("Registering extractor with %s : %s", url, result.text)
+            except Exception as exc:  # pylint: disable=broad-except
+                logger.exception('Error in registering extractor: ' + str(exc))
+
     # pylint: disable=no-self-use
     def status_update(self, status, resource, message):
         """Sends a status message.

pyclowder/files.py

@@ -58,6 +58,40 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
         raise
 
 
+# pylint: disable=too-many-arguments
+def download_v2(connector, host, token, fileid, intermediatefileid=None, ext=""):
+    """Download file to be processed from Clowder.
+
+    Keyword arguments:
+    connector -- connector information, used to get missing parameters and send status updates
+    host -- the clowder host, including http and port, should end with a /
+    key -- the secret key to login to clowder
+    fileid -- the file that is currently being processed
+    intermediatefileid -- either same as fileid, or the intermediate file to be used
+    ext -- the file extension, the downloaded file will end with this extension
+    """
+
+    connector.message_process({"type": "file", "id": fileid}, "Downloading file.")
+
+    # TODO: intermediateid doesn't really seem to be used here, can we remove entirely?
+    if not intermediatefileid:
+        intermediatefileid = fileid
+
+    url = '%sapi/v2/files/%s' % (host, intermediatefileid)
+    headers = {"Authorization": "Bearer " + token}
+    result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True, headers=headers)
+
+    (inputfile, inputfilename) = tempfile.mkstemp(suffix=ext)
+
+    try:
+        with os.fdopen(inputfile, "wb") as outputfile:
+            for chunk in result.iter_content(chunk_size=10*1024):
+                outputfile.write(chunk)
+        return inputfilename
+    except Exception:
+        os.remove(inputfilename)
+        raise
+
 def download_info(connector, host, key, fileid):
     """Download file summary metadata from Clowder.
 
@@ -69,12 +103,30 @@ def download_info(connector, host, key, fileid):
     """
 
     url = '%sapi/files/%s/metadata?key=%s' % (host, fileid, key)
+    headers = {"Authorization": "Bearer " + token}
 
     # fetch data
     result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True)
 
     return result.json()
 
+def download_info_v2(connector, host, token, fileid):
+    """Download file summary metadata from Clowder.
+
+    Keyword arguments:
+    connector -- connector information, used to get missing parameters and send status updates
+    host -- the clowder host, including http and port, should end with a /
+    key -- the secret key to login to clowder
+    fileid -- the file to fetch metadata of
+    """
+
+    url = '%sapi/v2/files/%s/metadata' % (host, fileid)
+    headers = {"Authorization": "Bearer " + token}
+    # fetch data
+    result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True, headers=headers)
+
+    return result.json()
+
 
 def download_metadata(connector, host, key, fileid, extractor=None):
     """Download file JSON-LD metadata from Clowder.

requirements.txt

@@ -22,3 +22,4 @@ requests-toolbelt==0.9.1
     # via pyclowder (setup.py)
 urllib3==1.26.8
     # via requests
+dotenv