Merge branch 'master' into dependabot/pip/certifi-2023.7.22

Author: max-zilla

.github/workflows/docker.yaml

@@ -3,7 +3,7 @@ name: Docker
 # This will run when:
 # - a new release is created, to make sure the right tags of the
 #   docker images are pushed (expects tags to be v1.8.4).
-# - when new code is pushed to main/develop to push the tags
+# - when new code is pushed to master/develop to push the tags
 #   latest and develop
 # - when a pull request is created and updated  to make sure the
 #   Dockerfile is still valid.
@@ -18,7 +18,7 @@ on:
 
   pull_request:
 
-# Certain actions will only run when this is the main repo.
+# Certain actions will only run when this is the master repo.
 env:
   MAIN_REPO: clowder-framework/pyclowder
   DOCKERHUB_ORG: clowder
@@ -37,7 +37,7 @@ jobs:
         include:
           - name: wordcount
             FOLDER: sample-extractors/wordcount
-            PLATFORM: "linux/amd64,linux/arm64"
+            PLATFORM: "linux/amd64"
     steps:
       - uses: actions/checkout@v2
 
@@ -56,7 +56,7 @@ jobs:
           # should we push to dockerhub, and is there a README
           DOCKERHUB_PUSH="false"
           DOCKERHUB_README="false"
-          if [ "$BRANCH" == "main" -a "${{ github.repository }}" == "${{ env.MAIN_REPO }}" ]; then
+          if [ "$BRANCH" == "master" -a "${{ github.repository }}" == "${{ env.MAIN_REPO }}" ]; then
             if [ "${{ secrets.DOCKERHUB_USERNAME }}" != "" -a "${{ secrets.DOCKERHUB_PASSWORD }}" != "" ]; then
               DOCKERHUB_PUSH="true"
               if [ -e "${{ matrix.FOLDER }}/README.md" ]; then
@@ -66,7 +66,7 @@ jobs:
           fi
 
           # calculate the version and all tags
-          if [ "$BRANCH" == "main" ]; then
+          if [ "$BRANCH" == "master" ]; then
             VERSION="$(awk '/"version":/ { print $2 }' ${{ matrix.FOLDER }}/extractor_info.json | sed 's/^.*"\([0-9\.]*\)".*$/\1/')"
             tags="latest"
             oldversion=""

CHANGELOG.md

@@ -5,6 +5,37 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](https://semver.org/).
 
+
+## 3.0.7 - 2023-10-11
+
+### Added
+
+- Modified v1 and v2 endpoints to ignore trailing slashes on Clowder host URLs.
+
+- ## 3.0.6 - 2023-10-10
+
+### Added
+
+- Allow folder id to be passed to api/v2 upload_to_dataset
+
+- ## 3.0.5 - 2023-10-09
+
+### Added
+
+- Support for deletion of individual files.
+
+## 3.0.4 - 2023-09-27
+
+### Fixed
+
+- Fixed host bug on v1 file thumbnail endpoint.
+
+## 3.0.3 - 2023-08-29
+
+### Added
+
+- Support for Clowder V2 thumbnails endpoints.
+
 ## 3.0.2 - 2023-07-20
 
 ### Added
@@ -46,6 +77,12 @@ This will result in only those donwloads to be counted by users, not extractors.
 
 - Ask not to track a download from an extractor.
 
+## Unreleased
+
+### Added
+- Add support for `EXTRACTOR_KEY` and `CLOWDER_EMAIL` environment variables to register
+an extractor for just one user.
+
 ## 2.6.0 - 2022-06-14
 
 This will change how clowder sees the extractors. If you have an extractor, and you specify

pyclowder/api/v1/datasets.py

@@ -2,7 +2,7 @@
 import logging
 import os
 import tempfile
-
+import posixpath
 import requests
 from pyclowder.client import ClowderClient
 from pyclowder.collections import get_datasets, get_child_collections, delete as delete_collection
@@ -22,7 +22,7 @@ def create_empty(connector, client, datasetname, description, parentid=None, spa
     """
     logger = logging.getLogger(__name__)
 
-    url = '%s/api/datasets/createempty?key=%s' % (client.host, client.key)
+    url = posixpath.join(client.host, 'api/datasets/createempty?key=%s' % client.key)
 
     if parentid:
         if spaceid:
@@ -61,7 +61,7 @@ def delete(connector, client, datasetid):
     client -- ClowderClient containing authentication credentials
     datasetid -- the dataset to delete
     """
-    url = "%s/api/datasets/%s?key=%s" % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, "api/datasets/%s?key=%s" % (datasetid, client.key))
 
     result = requests.delete(url, verify=connector.ssl_verify if connector else True)
     result.raise_for_status()
@@ -102,7 +102,7 @@ def download(connector, client, datasetid):
     connector.message_process({"type": "dataset", "id": datasetid}, "Downloading dataset.")
 
     # fetch dataset zipfile
-    url = '%s/api/datasets/%s/download?key=%s' % (client.host, datasetid,client.key)
+    url = posixpath.join(client.host, 'api/datasets/%s/download?key=%s' % datasetid,client.key)
     result = requests.get(url, stream=True,
                           verify=connector.ssl_verify if connector else True)
     result.raise_for_status()
@@ -124,7 +124,7 @@ def download_metadata(connector, client, datasetid, extractor=None):
     extractor -- extractor name to filter results (if only one extractor's metadata is desired)
     """
     filterstring = "" if extractor is None else "&extractor=%s" % extractor
-    url = '%s/api/datasets/%s/metadata?key=%s' % (client.host, datasetid, client.key + filterstring)
+    url = posixpath.join(client.host, 'api/datasets/%s/metadata.jsonld?key=%s' % (datasetid, client.key + filterstring))
 
     # fetch data
     result = requests.get(url, stream=True,
@@ -142,7 +142,7 @@ def get_info(connector, client, datasetid):
     datasetid -- the dataset to get info of
     """
 
-    url = "%s/api/datasets/%s?key=%s" % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, "api/datasets/%s?key=%s" % (datasetid, client.key))
 
     result = requests.get(url, verify=connector.ssl_verify if connector else True)
     result.raise_for_status()
@@ -157,7 +157,7 @@ def get_file_list(connector, client, datasetid):
     client -- ClowderClient containing authentication credentials
     datasetid -- the dataset to get filelist of
     """
-    url = "%s/api/datasets/%s/files?key=%s" % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, "api/datasets/%s/files?key=%s" % (datasetid, client.key))
 
     result = requests.get(url, verify=connector.ssl_verify if connector else True)
     result.raise_for_status()
@@ -175,7 +175,7 @@ def remove_metadata(connector, client, datasetid, extractor=None):
                     !!! ALL JSON-LD METADATA WILL BE REMOVED IF NO extractor PROVIDED !!!
     """
     filterstring = "" if extractor is None else "&extractor=%s" % extractor
-    url = '%s/api/datasets/%s/metadata?key=%s' % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, 'api/datasets/%s/metadata.jsonld?key=%s' % (datasetid, client.key))
 
     # fetch data
     result = requests.delete(url, stream=True, verify=connector.ssl_verify if connector else True)
@@ -192,7 +192,7 @@ def submit_extraction(connector, client, datasetid, extractorname):
     """
     headers = {'Content-Type': 'application/json'}
 
-    url = "%s/api/datasets/%s/extractions?key=%s" % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, "api/datasets/%s/extractions?key=%s" % (datasetid, client.key))
 
     result = requests.post(url,
                            headers=headers,
@@ -238,7 +238,7 @@ def upload_tags(connector, client, datasetid, tags):
     connector.status_update(StatusMessage.processing, {"type": "dataset", "id": datasetid}, "Uploading dataset tags.")
 
     headers = {'Content-Type': 'application/json'}
-    url = '%s/api/datasets/%s/tags?key=%s' % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, 'api/datasets/%s/tags?key=%s' % (datasetid, client.key))
     result = connector.post(url, headers=headers, data=json.dumps(tags),
                             verify=connector.ssl_verify if connector else True)
 
@@ -255,11 +255,45 @@ def upload_metadata(connector, client, datasetid, metadata):
     headers = {'Content-Type': 'application/json'}
     connector.message_process({"type": "dataset", "id": datasetid}, "Uploading dataset metadata.")
 
-    url = '%s/api/datasets/%s/metadata?key=%s' % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, 'api/datasets/%s/metadata.jsonld?key=%s' % (datasetid, client.key))
     result = requests.post(url, headers=headers, data=json.dumps(metadata),
                            verify=connector.ssl_verify if connector else True)
     result.raise_for_status()
 
+def upload_thumbnail(connector, host, key, datasetid, thumbnail):
+    """Upload thumbnail to Clowder.
+
+        Keyword arguments:
+        connector -- connector information, used to get missing parameters and send status updates
+        host -- the clowder host, including http and port, should end with a /
+        key -- the secret key to login to clowder
+        datasetid -- the dataset that the thumbnail should be associated with
+        thumbnail -- the file containing the thumbnail
+        """
+    logger = logging.getLogger(__name__)
+    logger.info("Upload thumbnails to datasets is not available in V1")
+
+
+def upload_preview(connector, host, key, datasetid, previewfile, previewmetadata=None, preview_mimetype=None,
+                   visualization_name=None, visualization_description=None, visualization_config_data=None,
+                   visualization_component_id=None):
+    """Upload preview to Clowder.
+
+    Keyword arguments:
+    connector -- connector information, used to get missing parameters and send status updates
+    host -- the clowder host, including http and port, should end with a /
+    key -- the secret key to login to clowder
+    datasetid -- the dataset that is currently being processed
+    previewfile -- the file containing the preview
+    previewmetadata -- any metadata to be associated with preview, can contain a section_id
+                    to indicate the section this preview should be associated with.
+    preview_mimetype -- (optional) MIME type of the preview file. By default, this is obtained from the
+                    file itself and this parameter can be ignored. E.g. 'application/vnd.clowder+custom+xml'
+    """
+
+    logger = logging.getLogger(__name__)
+    logger.info("Upload preview to datasets is not available in V1")
+
 
 # TODO not done yet, need more testing
 class DatasetsApi(object):
@@ -353,4 +387,4 @@ def add_metadata(self, dataset_id, metadata):
         try:
             return self.client.post("/datasets/%s/metadata" % dataset_id, metadata)
         except Exception as e:
-            logging.error("Error upload to dataset %s: %s" % (dataset_id, str(e)))
+            logging.error("Error upload to dataset %s: %s" % (dataset_id, str(e)))

pyclowder/api/v1/files.py

@@ -7,7 +7,7 @@
 import logging
 import os
 import tempfile
-
+import posixpath
 import requests
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
@@ -43,7 +43,7 @@ def get_download_url(connector, client, fileid, intermediatefileid=None, ext="")
     if not intermediatefileid:
         intermediatefileid = fileid
 
-    url = '%s/api/files/%s?key=%s' % (client.host, intermediatefileid, client.key)
+    url = posixpath.join(client.host, 'api/files/%s?key=%s' % (intermediatefileid, client.key))
     return url
 
 
@@ -65,7 +65,7 @@ def download(connector, client, fileid, intermediatefileid=None, ext=""):
     if not intermediatefileid:
         intermediatefileid = fileid
 
-    url = '%s/api/files/%s?key=%s' % (client.host, intermediatefileid, client.key)
+    url = posixpath.join(client.host, 'api/files/%s?key=%s' % (intermediatefileid, client.key))
     result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True)
 
     (inputfile, inputfilename) = tempfile.mkstemp(suffix=ext)
@@ -89,7 +89,7 @@ def download_info(connector, client, fileid):
     fileid -- the file to fetch metadata of
     """
 
-    url = '%s/api/files/%s/metadata?key=%s' % (client.host, fileid, client.key)
+    url = posixpath.join(client.host, 'api/files/%s/metadata?key=%s' % (fileid, client.key))
 
     # fetch data
     result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True)
@@ -121,14 +121,30 @@ def download_metadata(connector, client, fileid, extractor=None):
     """
 
     filterstring = "" if extractor is None else "&extractor=%s" % extractor
-    url = '%s/api/files/%s/metadata.jsonld?key=%s%s' % (client.host, fileid, client.key, filterstring)
+    url = posixpath.join(client.host, 'api/files/%s/metadata.jsonld?key=%s%s' % (fileid, client.key, filterstring))
 
     # fetch data
     result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True)
 
     return result
 
 
+def delete(connector, client, fileid):
+    """Delete file from Clowder.
+
+    Keyword arguments:
+    connector -- connector information, used to get missing parameters and send status updates
+    client -- ClowderClient containing authentication credentials
+    fileid -- the dataset to delete
+    """
+    url = posixpath.join(client.host, "api/files/%s?key=%s" % (fileid, client.key))
+
+    result = requests.delete(url, verify=connector.ssl_verify if connector else True)
+    result.raise_for_status()
+
+    return json.loads(result.text)
+
+
 def submit_extraction(connector, client, fileid, extractorname):
     """Submit file for extraction by given extractor.
 
@@ -139,7 +155,7 @@ def submit_extraction(connector, client, fileid, extractorname):
     extractorname -- registered name of extractor to trigger
     """
 
-    url = "%s/api/files/%s/extractions?key=%s" % (client.host, fileid, client.key)
+    url = posixpath.join(client.host, "api/files/%s/extractions?key=%s" % (fileid, client.key))
 
     result = connector.post(url,
                             headers={'Content-Type': 'application/json'},
@@ -213,7 +229,7 @@ def upload_metadata(connector, client, fileid, metadata):
     connector.message_process({"type": "file", "id": fileid}, "Uploading file metadata.")
 
     headers = {'Content-Type': 'application/json'}
-    url = '%s/api/files/%s/metadata.jsonld?key=%s' % (client.host, fileid, client.key)
+    url = posixpath.join(client.host, 'api/files/%s/metadata.jsonld?key=%s' % (fileid, client.key))
     result = connector.post(url, headers=headers, data=json.dumps(metadata),
                             verify=connector.ssl_verify if connector else True)
 
@@ -239,7 +255,7 @@ def upload_preview(connector, client, fileid, previewfile, previewmetadata=None,
     headers = {'Content-Type': 'application/json'}
 
     # upload preview
-    url = '%s/api/previews?key=%s' % (client.host, client.key)
+    url = posixpath.join(client.host, 'api/previews?key=%s' % client.key)
     with open(previewfile, 'rb') as filebytes:
         # If a custom preview file MIME type is provided, use it to generate the preview file object.
         if preview_mimetype is not None:
@@ -253,13 +269,13 @@ def upload_preview(connector, client, fileid, previewfile, previewmetadata=None,
 
     # associate uploaded preview with orginal file
     if fileid and not (previewmetadata and 'section_id' in previewmetadata and previewmetadata['section_id']):
-        url = '%s/api/files/%s/previews/%s?key=%s' % (client.host, fileid, previewid, client.key)
+        url = posixpath.join(client.host, 'api/files/%s/previews/%s?key=%s' % (fileid, previewid, client.key))
         result = connector.post(url, headers=headers, data=json.dumps({}),
                                 verify=connector.ssl_verify if connector else True)
 
     # associate metadata with preview
     if previewmetadata is not None:
-        url = '%s/api/previews/%s/metadata?key=%s' % (client.host, previewid, client.key)
+        url = posixpath.join(client.host, 'api/previews/%s/metadata?key=%s' % (previewid, client.key))
         result = connector.post(url, headers=headers, data=json.dumps(previewmetadata),
                                 verify=connector.ssl_verify if connector else True)
 
@@ -279,7 +295,7 @@ def upload_tags(connector, client, fileid, tags):
     connector.message_process({"type": "file", "id": fileid}, "Uploading file tags.")
 
     headers = {'Content-Type': 'application/json'}
-    url = '%s/api/files/%s/tags?key=%s' % (client.host, fileid, client.key)
+    url = posixpath.join(client.host, 'api/files/%s/tags?key=%s' % (fileid, client.key))
     result = connector.post(url, headers=headers, data=json.dumps(tags),
                             verify=connector.ssl_verify if connector else True)
 
@@ -295,18 +311,18 @@ def upload_thumbnail(connector, client, fileid, thumbnail):
     """
 
     logger = logging.getLogger(__name__)
-    url = client.host + 'api/fileThumbnail?key=' + client.key
+    url = posixpath.join(client.host, 'api/fileThumbnail?key=%s' % client.key)
 
     # upload preview
     with open(thumbnail, 'rb') as inputfile:
         result = connector.post(url, files={"File": inputfile}, verify=connector.ssl_verify if connector else True)
     thumbnailid = result.json()['id']
     logger.debug("thumbnail id = [%s]", thumbnailid)
 
-    # associate uploaded preview with orginal file/dataset
+    # associate uploaded preview with original file/dataset
     if fileid:
         headers = {'Content-Type': 'application/json'}
-        url = client.host + 'api/files/' + fileid + '/thumbnails/' + thumbnailid + '?key=' + client.key
+        url = posixpath.join(client.host, 'api/files/%s/thumbnails/%s?key=%s' % (fileid, thumbnailid, client.key))
         connector.post(url, headers=headers, data=json.dumps({}), verify=connector.ssl_verify if connector else True)
 
     return thumbnailid
@@ -336,7 +352,7 @@ def upload_to_dataset(connector, client, datasetid, filepath, check_duplicate=Fa
         if filepath.startswith(connector.mounted_paths[source_path]):
             return _upload_to_dataset_local(connector, client, datasetid, filepath)
 
-    url = '%s/api/uploadToDataset/%s?key=%s' % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, 'api/uploadToDataset/%s?key=%s' % (datasetid, client.key))
 
     if os.path.exists(filepath):
         filename = os.path.basename(filepath)
@@ -365,7 +381,7 @@ def _upload_to_dataset_local(connector, client, datasetid, filepath):
     """
 
     logger = logging.getLogger(__name__)
-    url = '%s/api/uploadToDataset/%s?key=%s' % (client.host, datasetid, client.key)
+    url = posixpath.join(client.host, 'api/uploadToDataset/%s?key=%s' % (datasetid, client.key))
 
     if os.path.exists(filepath):
         # Replace local path with remote path before uploading