Merge branch 'master' into 59-change-context-type-for-v2

Author: tcnichol

CHANGELOG.md

@@ -15,6 +15,14 @@ This version adds Clowder 2 support and removes the old method of extractor regi
 - remove RABBITMQ_EXCHANGE parameter and REGISTRATION_URL parameter.
 - remove DatasetsAPI and object-oriented ClowderClient.
 
+## 2.7.0 - 2023-02-14
+
+When extractors download a file from clowder it will ask clowder to not track that download.
+This will result in only those donwloads to be counted by users, not extractors.
+
+### Changed
+- Ask not to track a download from an extractor.
+
 ## 2.6.0 - 2022-06-14
 
 This will change how clowder sees the extractors. If you have an extractor, and you specify

README.md

@@ -15,7 +15,7 @@ create new extractors.
 Install using pip (for most recent versions see: https://pypi.org/project/pyclowder/):
 
 ```
-pip install pyclowder==2.6.0
+pip install pyclowder==2.7.0
 ```
 
 Install pyClowder on your system by cloning this repo:

description.rst

@@ -14,7 +14,7 @@ Installation
 Install using pip (for most recent versions see: https://pypi.org/project/pyclowder/):
 
 ```
-pip install pyclowder==2.6.0
+pip install pyclowder==2.7.0
 ```
 
 Install pyClowder on your system by cloning this repo:

docs/source/conf.py

@@ -57,9 +57,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.6'
+version = u'2.7'
 # The full version, including alpha/beta/rc tags.
-release = u'2.6.0'
+release = u'2.7.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pyclowder/connectors.py

@@ -340,7 +340,7 @@ def _prepare_dataset(self, host, secret_key, resource):
             for ds_file in missing_files:
                 # Download file to temp directory
                 inputfile = pyclowder.files.download(self, host, secret_key, ds_file['id'], ds_file['id'],
-                                                     ds_file['file_ext'])
+                                                     ds_file['file_ext'], tracking=False)
                 # Also get file metadata in format expected by extractor
                 (file_md_dir, file_md_tmp) = self._download_file_metadata(host, secret_key, ds_file['id'],
                                                                           ds_file['filepath'])
@@ -426,8 +426,9 @@ def _process_message(self, body):
                                 file_path = self._check_for_local_file(file_metadata)
                                 if not file_path:
                                     file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
-                                                                             resource["intermediate_id"],
-                                                                             resource["file_ext"])
+                                                                         resource["intermediate_id"],
+                                                                         resource["file_ext"],
+                                                                         tracking=False)
                                 else:
                                     found_local = True
                                 resource['local_paths'] = [file_path]

pyclowder/files.py

@@ -36,7 +36,7 @@ def get_download_url(connector, host, key, fileid, intermediatefileid=None, ext=
     return download_url
 
 # pylint: disable=too-many-arguments
-def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
+def download(connector, host, key, fileid, intermediatefileid=None, ext="", tracking=True):
     """Download file to be processed from Clowder.
 
     Keyword arguments:
@@ -46,6 +46,7 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
     fileid -- the file that is currently being processed
     intermediatefileid -- either same as fileid, or the intermediate file to be used
     ext -- the file extension, the downloaded file will end with this extension
+    tracking -- should the download action be tracked
     """
     client = ClowderClient(host=host, key=key)
     if clowder_version == 2:
@@ -54,7 +55,6 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
         inputfilename = v1files.download(connector, client, fileid, intermediatefileid, ext)
     return inputfilename
 
-
 def download_info(connector, host, key, fileid):
     """Download file summary metadata from Clowder.
 

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile
 #
-certifi==2021.10.8
+certifi==2022.12.7
     # via requests
 charset-normalizer==2.0.10
     # via requests

sample-extractors/wordcount/requirements.txt

@@ -1 +1 @@
-pyclowder==2.6.0
+pyclowder==2.7.0

version.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-VERSION=${1:-2.5.1}
-
+VERSION=$(awk '/^## / { print $2 }' CHANGELOG.md | head -1)
 MAJOR=${VERSION%.*}
 
 sed -i~ "s/## unreleased.*/## ${VERSION} - $(date +'%Y-%m-%d')/i" CHANGELOG.md