don't track extractor downloads (fixes #55)

Author: robkooper

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 2.7.0 - 2023-02-14
+
+When extractors download a file from clowder it will ask clowder to not track that download.
+This will result in only those donwloads to be counted by users, not extractors.
+
+### Changed
+- Ask not to track a download from an extractor.
+
 ## 2.6.0 - 2022-06-14
 
 This will change how clowder sees the extractors. If you have an extractor, and you specify

pyclowder/connectors.py

@@ -332,7 +332,7 @@ def _prepare_dataset(self, host, secret_key, resource):
             for ds_file in missing_files:
                 # Download file to temp directory
                 inputfile = pyclowder.files.download(self, host, secret_key, ds_file['id'], ds_file['id'],
-                                                     ds_file['file_ext'])
+                                                     ds_file['file_ext'], tracking=False)
                 # Also get file metadata in format expected by extractor
                 (file_md_dir, file_md_tmp) = self._download_file_metadata(host, secret_key, ds_file['id'],
                                                                           ds_file['filepath'])
@@ -425,7 +425,8 @@ def _process_message(self, body):
                                 if not file_path:
                                     file_path = pyclowder.files.download(self, host, secret_key, resource["id"],
                                                                          resource["intermediate_id"],
-                                                                         resource["file_ext"])
+                                                                         resource["file_ext"],
+                                                                         tracking=False)
                                 else:
                                     found_local = True
                                 resource['local_paths'] = [file_path]

pyclowder/files.py

@@ -25,7 +25,7 @@
 
 
 # pylint: disable=too-many-arguments
-def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
+def download(connector, host, key, fileid, intermediatefileid=None, ext="", tracking=True):
     """Download file to be processed from Clowder.
 
     Keyword arguments:
@@ -35,6 +35,7 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
     fileid -- the file that is currently being processed
     intermediatefileid -- either same as fileid, or the intermediate file to be used
     ext -- the file extension, the downloaded file will end with this extension
+    tracking -- should the download action be tracked
     """
 
     connector.message_process({"type": "file", "id": fileid}, "Downloading file.")
@@ -43,7 +44,7 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
     if not intermediatefileid:
         intermediatefileid = fileid
 
-    url = '%sapi/files/%s?key=%s' % (host, intermediatefileid, key)
+    url = '%sapi/files/%s?key=%s&tracking=%s' % (host, intermediatefileid, key, str(tracking).lower())
     result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True)
 
     (inputfile, inputfilename) = tempfile.mkstemp(suffix=ext)