Merge pull request #76 from clowder-framework/72-add-same-v2-visualization-to-pyclowder-dataset-class

Adding upload_preview method for dataset

Author: max-zilla

pyclowder/api/v2/datasets.py

@@ -9,6 +9,7 @@
 import tempfile
 
 import requests
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from pyclowder.client import ClowderClient
 from pyclowder.collections import get_datasets, get_child_collections, delete as delete_collection
@@ -233,3 +234,85 @@ def upload_metadata(connector, client, datasetid, metadata):
                            verify=connector.ssl_verify if connector else True)
     result.raise_for_status()
 
+def upload_preview(connector, client, datasetid, previewfile, previewmetadata=None, preview_mimetype=None,
+                   visualization_name=None, visualization_description=None, visualization_config_data=None,
+                   visualization_component_id=None):
+    """Upload visualization to Clowder.
+
+    Keyword arguments:
+    connector -- connector information, used to get missing parameters and send status updates
+    client -- ClowderClient containing authentication credentials
+    datsetid -- the dataset that is currently being processed
+    previewfile -- the file containing the preview
+    previewmetadata -- any metadata to be associated with preview, can contain a section_id
+                    to indicate the section this preview should be associated with.
+    preview_mimetype -- (optional) MIME type of the preview file. By default, this is obtained from the
+                    file itself and this parameter can be ignored. E.g. 'application/vnd.clowder+custom+xml'
+    """
+
+    connector.message_process({"type": "dataset", "id": datasetid}, "Uploading dataset preview.")
+    logger = logging.getLogger(__name__)
+
+    preview_id = None
+    visualization_config_id = None
+
+    if os.path.exists(previewfile):
+
+        # upload visualization URL
+        visualization_config_url = '%s/api/v2/visualizations/config' % client.host
+
+        if visualization_config_data is None:
+            visualization_config_data = dict()
+
+        payload = json.dumps({
+            "resource": {
+                "collection": "datasets",
+                "resource_id": datasetid
+            },
+            "client": client.host,
+            "parameters": visualization_config_data,
+            "visualization_mimetype": preview_mimetype,
+            "visualization_component_id": visualization_component_id
+        })
+
+        headers = {
+            "X-API-KEY": client.key,
+            "Content-Type": "application/json"
+        }
+
+        response = connector.post(visualization_config_url, headers=headers, data=payload,
+                                  verify=connector.ssl_verify if connector else True)
+
+        if response.status_code == 200:
+            visualization_config_id = response.json()['id']
+            logger.debug("Uploaded visualization config ID = [%s]", visualization_config_id)
+        else:
+            logger.error("An error occurred when uploading visualization config to dataset: " + datasetid)
+
+        if visualization_config_id is not None:
+
+            # upload visualization URL
+            visualization_url = '%s/api/v2/visualizations?name=%s&description=%s&config=%s' % (
+                client.host, visualization_name, visualization_description, visualization_config_id)
+
+            filename = os.path.basename(previewfile)
+            if preview_mimetype is not None:
+                multipart_encoder_object = MultipartEncoder(
+                    fields={'file': (filename, open(previewfile, 'rb'), preview_mimetype)})
+            else:
+                multipart_encoder_object = MultipartEncoder(fields={'file': (filename, open(previewfile, 'rb'))})
+            headers = {'X-API-KEY': client.key,
+                       'Content-Type': multipart_encoder_object.content_type}
+            response = connector.post(visualization_url, data=multipart_encoder_object, headers=headers,
+                                      verify=connector.ssl_verify if connector else True)
+
+            if response.status_code == 200:
+                preview_id = response.json()['id']
+                logger.debug("Uploaded visualization data ID = [%s]", preview_id)
+            else:
+                logger.error("An error occurred when uploading the visualization data to dataset: " + datasetid)
+    else:
+        logger.error("Visualization data file not found")
+
+    return preview_id
+

pyclowder/datasets.py

@@ -16,6 +16,12 @@
 from pyclowder.utils import StatusMessage
 
 clowder_version = int(os.getenv('CLOWDER_VERSION', '1'))
+# Import dataset API methods based on Clowder version
+if clowder_version == 2:
+    import pyclowder.api.v2.datasets as datasets
+else:
+    import pyclowder.api.v1.datasets as datasets
+
 
 def create_empty(connector, host, key, datasetname, description, parentid=None, spaceid=None):
     """Create a new dataset in Clowder.
@@ -30,10 +36,7 @@ def create_empty(connector, host, key, datasetname, description, parentid=None,
     spaceid -- id of the space to add dataset to
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        datasetid = v2datasets.create_empty(connector, client, datasetname, description, parentid, spaceid)
-    else:
-        datasetid = v1datasets.create_empty(connector, client, datasetname, description, parentid, spaceid)
+    datasetid = datasets.create_empty(connector, client, datasetname, description, parentid, spaceid)
     return datasetid
 
 
@@ -47,10 +50,7 @@ def delete(connector, host, key, datasetid):
     datasetid -- the dataset to delete
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        result = v2datasets.delete(connector, client, datasetid)
-    else:
-        result = v1datasets.delete(connector, client, datasetid)
+    result = datasets.delete(connector, client, datasetid)
     result.raise_for_status()
 
     return json.loads(result.text)
@@ -91,10 +91,7 @@ def download(connector, host, key, datasetid):
     datasetid -- the file that is currently being processed
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        zipfile = v2datasets.download(connector, client, datasetid)
-    else:
-        zipfile = v1datasets.download(connector, client, datasetid)
+    zipfile = datasets.download(connector, client, datasetid)
     return zipfile
 
 
@@ -109,12 +106,8 @@ def download_metadata(connector, host, key, datasetid, extractor=None):
     extractor -- extractor name to filter results (if only one extractor's metadata is desired)
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        result_json = v2datasets.download_metadata(connector, client, datasetid, extractor)
-        return result_json
-    else:
-        result_json = v1datasets.download_metadata(connector, client, datasetid, extractor)
-        return result_json
+    result_json = datasets.download_metadata(connector, client, datasetid, extractor)
+    return result_json
 
 
 def get_info(connector, host, key, datasetid):
@@ -127,10 +120,7 @@ def get_info(connector, host, key, datasetid):
     datasetid -- the dataset to get info of
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        info = v2datasets.get_info(connector, client, datasetid)
-    else:
-        info = v1datasets.get_info(connector, client, datasetid)
+    info = datasets.get_info(connector, client, datasetid)
     return info
 
 
@@ -144,10 +134,7 @@ def get_file_list(connector, host, key, datasetid):
     datasetid -- the dataset to get filelist of
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        file_list = v2datasets.get_file_list(connector, client, datasetid)
-    else:
-        file_list = v1datasets.get_file_list(connector, client, datasetid)
+    file_list = datasets.get_file_list(connector, client, datasetid)
     return file_list
 
 
@@ -163,10 +150,7 @@ def remove_metadata(connector, host, key, datasetid, extractor=None):
                     !!! ALL JSON-LD METADATA WILL BE REMOVED IF NO extractor PROVIDED !!!
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        v2datasets.remove_metadata(connector, client, datasetid, extractor)
-    else:
-        v1datasets.remove_metadata(connector, client, datasetid, extractor)
+    datasets.remove_metadata(connector, client, datasetid, extractor)
 
 
 def submit_extraction(connector, host, key, datasetid, extractorname):
@@ -180,10 +164,7 @@ def submit_extraction(connector, host, key, datasetid, extractorname):
     extractorname -- registered name of extractor to trigger
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        result_status_code = v2datasets.submit_extraction(connector, client, datasetid, extractorname)
-    else:
-        result_status_code = v1datasets.submit_extraction(connector, client, datasetid, extractorname)
+    return datasets.submit_extraction(connector, client, datasetid, extractorname)
 
 
 def submit_extractions_by_collection(connector, host, key, collectionid, extractorname, recursive=True):
@@ -242,7 +223,30 @@ def upload_metadata(connector, host, key, datasetid, metadata):
     metadata -- the metadata to be uploaded
     """
     client = ClowderClient(host=host, key=key)
-    if clowder_version == 2:
-        v2datasets.upload_metadata(connector, client, datasetid, metadata)
-    else:
-        v1datasets.upload_metadata(connector, client, datasetid, metadata)
+    datasets.upload_metadata(connector, client, datasetid, metadata)
+
+
+def upload_preview(connector, host, key, datasetid, previewfile, previewmetadata=None, preview_mimetype=None,
+                   visualization_name=None, visualization_description=None, visualization_config_data=None,
+                   visualization_component_id=None):
+    """Upload preview to Clowder.
+
+    Keyword arguments:
+    connector -- connector information, used to get missing parameters and send status updates
+    host -- the clowder host, including http and port, should end with a /
+    key -- the secret key to login to clowder
+    datasetid -- the dataset that is currently being processed
+    previewfile -- the file containing the preview
+    previewmetadata -- any metadata to be associated with preview, can contain a section_id
+                    to indicate the section this preview should be associated with.
+    preview_mimetype -- (optional) MIME type of the preview file. By default, this is obtained from the
+                    file itself and this parameter can be ignored. E.g. 'application/vnd.clowder+custom+xml'
+    """
+
+    client = ClowderClient(host=host, key=key)
+    preview_id = datasets.upload_preview(connector, client, datasetid, previewfile, previewmetadata, preview_mimetype,
+                                         visualization_name=visualization_name,
+                                         visualization_description=visualization_description,
+                                         visualization_config_data=visualization_config_data,
+                                         visualization_component_id=visualization_component_id)
+    return preview_id

sample-extractors/test-dataset-extractor/preview_file.jpeg

@@ -65,6 +65,19 @@ def process_message(self, connector, host, secret_key, resource, parameters):
         else:
             logger.info("Metadata: %s", dataset_metadata)
 
+        # Upload a preview to dataset
+        # Local file path to file which you want to upload to dataset for preview
+        preview_file_path = os.path.join(os.getcwd(), 'preview_file.jpeg')
+        preview_id = pyclowder.datasets.upload_preview(connector, host, secret_key, dataset_id, preview_file_path, None,
+                                                       "image/jpeg", visualization_name="test-dataset-extractor",
+                                                       visualization_component_id="basic-image-component")
+        if preview_id is None:
+            logger.info("Preview upload failed")
+        else:
+            logger.info("Preview %s uploaded to dataset successfully ", preview_id)
+
+
+
 
 if __name__ == "__main__":
     extractor = TestDatasetExtractor()