Merge branch 'master' into clowder-internal-url

Author: max-zilla

.github/workflows/ci.yaml

@@ -0,0 +1,46 @@
+name: Python package
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        # This path is specific to Ubuntu
+        path: ~/.cache/pip
+        # Look to see if there is a cache hit for the corresponding requirements file
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+          ${{ runner.os }}-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest pipenv
+        pip install -r requirements.txt
+
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings.
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics
+
+#    - name: Test with pytest
+#      run: |
+#        pytest

.github/workflows/pypi.yaml

@@ -0,0 +1,46 @@
+name: Publish Python package
+
+on:
+   push:
+     tags:
+
+jobs:
+  publish:
+    name: Build and publish python packages
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: python -m pip install --upgrade pip setuptools wheel
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Build dist file
+        run: |
+          python setup.py sdist bdist_wheel
+
+      - name: Publish distribution to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.pypi_password }}

CHANGELOG.md

@@ -4,10 +4,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/) 
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## Unreleased
+## 2.3.1 - 2020-09-18
+
+With this version we no longer gurantee support for versions of python below 3.
+
+### Fixed
+- There was an issue where status messages could cause an exception. This would prevent most extractors from running correctly.
+
+## 2.3.0 - 2020-09-15
+
+**CRITICAL BUG IN THIS VERSION. PLEASE USE VERSION 2.3.1**
+
+Removed develop branch, all pull requests will need to be against master from now
+forward. Please update version number in setup.py in each PR.
+
+From this version no more docker images are build, please use pip install to
+install pyclowder.
 
 ### Added
 - Simple extractors now support datasets, can also create new datasets.
+- Ability to add tags from simple extractor to files and datasets.
+- Ability to add additional files (outputs) to dataset in simple extractor.
+- Use pipenv to manage dependencies.
+- Add job_id to each status message returned by pyclowder.
+- PyClowderExtractionAbort to indicate the message shoudl not be retried.
+
+### Changed
+- Better handling of status messages
 
 ## 2.2.3 - 2019-10-14
 

Pipfile

@@ -0,0 +1,15 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+pika = "*"
+requests = "*"
+requests-toolbelt = "*"
+pyyaml = "==5.1"
+
+[requires]
+python_version = "3.7"

Pipfile.lock

@@ -8,7 +8,6 @@
 import requests
 
 from pyclowder.client import ClowderClient
-from pyclowder.utils import StatusMessage
 
 
 def create_empty(connector, host, key, collectionname, description, parentid=None, spaceid=None):
@@ -121,8 +120,7 @@ def upload_preview(connector, host, key, collectionid, previewfile, previewmetad
                     section this preview should be associated with.
     """
 
-    connector.status_update(StatusMessage.processing, {"type": "collection", "id": collectionid},
-                            "Uploading collection preview.")
+    connector.message_process({"type": "collection", "id": collectionid}, "Uploading collection preview.")
 
     logger = logging.getLogger(__name__)
     headers = {'Content-Type': 'application/json'}