Merge branch 'master' into fix-threading

Author: robkooper

.github/workflows/ci.yaml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/pypi.yaml

@@ -1,16 +1,25 @@
 name: Publish Python package
 
 on:
-   push:
-     tags:
+  push:
+    branches:
+      - main
+      - master
 
 jobs:
   publish:
     name: Build and publish python packages
     runs-on: ubuntu-18.04
+
     steps:
       - uses: actions/checkout@v2
 
+      - name: get release info
+        id: release_info
+        run: |
+          version="$(awk '/^## / { print tolower($2) }' CHANGELOG.md | head -1)"
+          echo "::set-output name=version::$version"
+
       - name: Set up Python 3.7
         uses: actions/setup-python@v2
         with:
@@ -40,7 +49,7 @@ jobs:
           python setup.py sdist bdist_wheel
 
       - name: Publish distribution to PyPI
-        if: startsWith(github.ref, 'refs/tags')
+        if: steps.release_info.outputs.version != 'unreleased'
         uses: pypa/gh-action-pypi-publish@master
         with:
           password: ${{ secrets.pypi_password }}

.github/workflows/release.yaml

@@ -0,0 +1,38 @@
+name: Create Release
+
+on:
+  push:
+    branches:
+      - main
+      - master
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: get release info
+        id: release_info
+        run: |
+          version="$(awk '/^## / { print tolower($2) }' CHANGELOG.md | head -1)"
+          changelog="$(sed -e "1,/^## ${version}/d" -e "/^## /,\$d" CHANGELOG.md)"
+          changelog="${changelog//'%'/'%25'}"
+          changelog="${changelog//$'\n'/'%0A'}"
+          changelog="${changelog//$'\r'/'%0D'}"
+          echo "::set-output name=version::$version"
+          echo "::set-output name=changelog::$changelog"
+
+      - name: create release
+        if: steps.release_info.outputs.version != 'unreleased'
+        uses: actions/create-release@v1
+        id: create_release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.release_info.outputs.version }}
+          release_name: Release ${{ steps.release_info.outputs.version }}
+          body: ${{ steps.release_info.outputs.changelog }}

CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/) 
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## unreleased
+
+### Changed
+- updated all of the requirements to latest versions
+- updated github actions to automatically create releases
+
+## 2.4.0 - 2021-02-22
+
+### Changed
+- clowder is no longer the default exchange. Exchanges are no longer used and
+  this is deprecated.
+- fix check for thread is_alive, fixes warning in python 3.9
+
+### Removed
+- Removed the extractors.<queue_name> since it was not used.
+
 ## 2.3.4 - 2020-10-04
 
 ### Fixed

docs/source/conf.py

@@ -57,9 +57,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.1'
+version = u'2.4'
 # The full version, including alpha/beta/rc tags.
-release = u'2.1.1'
+release = u'2.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pyclowder/connectors.py

@@ -661,7 +661,6 @@ def connect(self):
 
         # declare the queue in case it does not exist
         self.channel.queue_declare(queue=self.rabbitmq_queue, durable=True)
-        self.channel.queue_declare(queue='extractors.' + self.rabbitmq_queue, durable=True)
         self.channel.queue_declare(queue='error.'+self.rabbitmq_queue, durable=True)
 
         # register with an exchange
@@ -884,7 +883,7 @@ def start_thread(self, json_body):
 
     def is_finished(self):
         with self.lock:
-            return self.thread and not self.thread.isAlive() and self.finished and len(self.messages) == 0
+            return self.thread and not self.thread.is_alive() and self.finished and len(self.messages) == 0
 
     def process_messages(self, channel, rabbitmq_queue):
         while self.messages:

pyclowder/extractors.py

@@ -63,7 +63,7 @@ def __init__(self):
         if not rabbitmq_queuename:
             rabbitmq_queuename = self.extractor_info['name']
         rabbitmq_uri = os.getenv('RABBITMQ_URI', "amqp://guest:[email protected]/%2f")
-        rabbitmq_exchange = os.getenv('RABBITMQ_EXCHANGE', "clowder")
+        rabbitmq_exchange = os.getenv('RABBITMQ_EXCHANGE', "")
         clowder_url = os.getenv("CLOWDER_URL", "")
         registration_endpoints = os.getenv('REGISTRATION_ENDPOINTS', "")
         logging_config = os.getenv("LOGGING")
@@ -143,7 +143,7 @@ def start(self):
                     for key, value in self.extractor_info['process'].items():
                         for mt in value:
                             # Replace trailing '*' with '#'
-                            mt = re.sub('(\*$)', '#', mt)
+                            mt = re.sub(r'(\*$)', '#', mt)
                             if mt.find('*') > -1:
                                 logger.error("Invalid '*' found in rabbitmq_key: %s" % mt)
                             else:

pyclowder/files.py

@@ -47,6 +47,7 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext=""):
     result = connector.get(url, stream=True, verify=connector.ssl_verify if connector else True)
 
     (inputfile, inputfilename) = tempfile.mkstemp(suffix=ext)
+
     try:
         with os.fdopen(inputfile, "wb") as outputfile:
             for chunk in result.iter_content(chunk_size=10*1024):

requirements.txt

@@ -1,5 +1,5 @@
-enum34==1.1.6
-pika==1.1.0
-PyYAML==5.1
-requests==2.24.0
+enum34==1.1.10
+pika==1.2.0
+PyYAML==5.4.1
+requests==2.26.0
 requests-toolbelt==0.9.1

setup.py

@@ -9,7 +9,7 @@ def description():
 
 
 setup(name='pyclowder',
-      version='2.3.4',
+      version='2.4.0',
       packages=find_packages(),
       description='Python SDK for the Clowder Data Management System',
       long_description=description(),