[v2.0.1] CircleCI, 500 Error code, Readme updates (#27)

fatihkurtoglu · web-flow · commit 68a89dea6011 · 2021-04-07T14:44:14.000-07:00
* readme updates

* version bumped

* docstring tools for developer guide

* circleci configurations

* pytest test key validation

* added 500 error code to retry

* updated publish script
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,103 @@
+# CircleCI jobs are only enabled to on Pull Requests and commits to master branch.
+# "Only build pull requests" enabled in Project's Advanced Settings.
+version: 2.1
+jobs:
+  build_test:
+    docker:
+      - image: cimg/python:3.6
+    resource_class: small
+    steps:
+      - checkout  # checkout source code to working directory
+      - run:
+          name: Install Environment Dependencies
+          command: |  # install env dependencies
+            set -e
+            pip install --upgrade pip
+            pip install -r docs/dev_requirements.txt
+      - run:
+          name: Black Formatting Check  # Only validation, without re-formatting
+          command: |
+            black --check -t py36 .
+      - run:
+          name: isort Import Ordering Check  # Only validation, without re-formatting
+          command: |
+            isort --check-only --profile black .
+      - run:
+          name: Flake8 Lint Check  # Uses setup.cfg for configuration
+          command: |
+            flake8 . --count --statistics
+      - run:
+          name: Pylint Lint Check # Uses .pylintrc for configuration
+          command: |
+            pylint scaleapi
+      - run:
+          name: Build Package # create whl and install package
+          command: |
+            set -e
+            python setup.py sdist bdist_wheel
+            pip install --no-cache-dir dist/*.whl
+      - run:
+          name: Pytest Test Cases
+          command: |  # Run test suite, uses SCALE_TEST_API_KEY env variable
+            pytest -v
+      - run:
+          name: Twine PyPI Check
+          command: |  # Validate distribution and setup.py configuration
+            twine check --strict dist/*
+  pypi_publish:
+    docker:
+      - image: cimg/python:3.6
+    steps:
+      - checkout  # checkout source code to working directory
+      - run:
+          name: Validate Tag Version  # Check if the tag name matches the package version
+          command: |
+            PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//')
+
+            if [[ "$CIRCLE_TAG" != "v${PKG_VERSION}" ]]; then
+                echo "ERROR: Tag name ($CIRCLE_TAG) must match package version (v${PKG_VERSION})."
+                exit 1;
+            fi
+      - run:
+          name: Validate SDK Version Increment  # Check if the version is already on PyPI
+          command: |
+            PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//')
+
+            if pip install "scaleapi>=${PKG_VERSION}" > /dev/null 2>&1;
+            then
+                echo "ERROR: You need to increment to a new version before publishing!"
+                echo "Version (${PKG_VERSION}) already exists on PyPI."
+                exit 1;
+            fi
+      - run:
+          name: Install Environment Dependencies
+          command: |  # install env dependencies
+            set -e
+            pip install --upgrade pip
+            pip install twine
+      - run:
+          name: Build and Validate
+          command: |  # create whl, validate with twine
+            set -e
+            python setup.py sdist bdist_wheel
+            twine check --strict dist/*
+      - run:
+          name: Publish to PyPI
+          command: |
+            if test -z "${TWINE_USERNAME}" || test -z "${TWINE_PASSWORD}" ; then
+                echo "ERROR: Please assign TWINE_USERNAME and TWINE_PASSWORD as environment variables"
+                exit 1
+            fi
+            twine upload dist/*
+workflows:
+  build_test_publish:
+    jobs:
+      - build_test
+      - pypi_publish:
+          requires:
+            - build_test
+          filters:
+            tags:
+              only: /^v\d+\.\d+\.\d+$/  # Runs only for tags with the format [v1.2.3]
+            branches:
+              ignore: /.*/  # Runs for none of the branches
diff --git a/README.rst b/README.rst
@@ -31,7 +31,8 @@ _____
 .. code-block:: python
 
     import scaleapi
-    client = scaleapi.ScaleClient('YOUR_API_KEY_HERE')
+
+    client = scaleapi.ScaleClient("YOUR_API_KEY_HERE")
 
 Tasks
 _____
@@ -62,22 +63,28 @@ __ https://docs.scale.com/reference
 
     from scaleapi.tasks import TaskType
 
-    client.create_task(
-        TaskType.ImageAnnotation,
-        project = 'test_project',
+    payload = dict(
+        project = "test_project",
         callback_url = "http://www.example.com/callback",
-        instruction= "Draw a box around each baby cow and big cow.",
+        instruction = "Draw a box around each baby cow and big cow.",
         attachment_type = "image",
         attachment = "http://i.imgur.com/v4cBreD.jpg",
+        unique_id = "c235d023af73",
         geometries = {
             "box": {
-              "objects_to_annotate": ["Baby Cow", "Big Cow"],
-              "min_height": 10,
-              "min_width": 10
+                "objects_to_annotate": ["Baby Cow", "Big Cow"],
+                "min_height": 10,
+                "min_width": 10,
             }
-        }
+        },
     )
 
+    try:
+        client.create_task(TaskType.ImageAnnotation, **payload)
+    except ScaleDuplicateTask as err:
+        print(err.message)  # If unique_id is already used for a different task
+
+
 Retrieve a task
 ^^^^^^^^^^^^^^^
 
@@ -87,8 +94,8 @@ __ https://docs.scale.com/reference#retrieve-tasks
 
 .. code-block :: python
 
-    task = client.get_task('30553edd0b6a93f8f05f0fee')
-    print(task.status)  # Task status ('pending', 'completed', 'error', 'canceled')
+    task = client.get_task("30553edd0b6a93f8f05f0fee")
+    print(task.status)  # Task status ("pending", "completed", "error", "canceled")
     print(task.response) # If task is complete
 
 List Tasks
@@ -100,9 +107,9 @@ Retrieve a list of `Task` objects, with filters for: ``project_name``, ``batch_n
 
 ``get_tasks()`` is a **generator** method and yields ``Task`` objects.
 
-`A generator is another type of function, returns an iterable that you can loop over like a list.
+*A generator is another type of function, returns an iterable that you can loop over like a list.
 However, unlike lists, generators do not store the content in the memory.
-That helps you to process a large number of objects without increasing memory usage.`
+That helps you to process a large number of objects without increasing memory usage.*
 
 If you will iterate through the tasks and process them once, using a generator is the most efficient method.
 However, if you need to process the list of tasks multiple times, you can wrap the generator in a ``list(...)``
@@ -157,9 +164,9 @@ __ https://docs.scale.com/reference#batch-creation
 .. code-block:: python
 
     client.create_batch(
-        project = 'test_project',
+        project = "test_project",
         callback = "http://www.example.com/callback",
-        name = 'batch_name_01_07_2021'
+        name = "batch_name_01_07_2021"
     )
 
 Finalize Batch
@@ -171,7 +178,11 @@ __ https://docs.scale.com/reference#batch-finalization
 
 .. code-block:: python
 
-    client.finalize_batch(batch_name = 'batch_name_01_07_2021')
+    client.finalize_batch(batch_name="batch_name_01_07_2021")
+
+    # Alternative method
+    batch = client.get_batch(batch_name="batch_name_01_07_2021")
+    batch.finalize()
 
 Check Batch Status
 ^^^^^^^^^^^^^^^^^^
@@ -182,10 +193,10 @@ __ https://docs.scale.com/reference#batch-status
 
 .. code-block:: python
 
-    client.batch_status(batch_name = 'batch_name_01_07_2021')
+    client.batch_status(batch_name = "batch_name_01_07_2021")
 
     # Alternative via Batch.get_status()
-    batch = client.get_batch('batch_name_01_07_2021')
+    batch = client.get_batch("batch_name_01_07_2021")
     batch.get_status() # Refreshes tasks_{status} attributes of Batch
     print(batch.tasks_pending, batch.tasks_completed)
 
@@ -198,7 +209,7 @@ __ https://docs.scale.com/reference#batch-retrieval
 
 .. code-block:: python
 
-    client.get_batch(batch_name = 'batch_name_01_07_2021')
+    client.get_batch(batch_name = "batch_name_01_07_2021")
 
 List Batches
 ^^^^^^^^^^^^
@@ -207,9 +218,9 @@ Retrieve a list of Batches. Optional parameters are ``project_name``, ``batch_st
 
 ``get_batches()`` is a **generator** method and yields ``Batch`` objects.
 
-`A generator is another type of function, returns an iterable that you can loop over like a list.
+*A generator is another type of function, returns an iterable that you can loop over like a list.
 However, unlike lists, generators do not store the content in the memory.
-That helps you to process a large number of objects without increasing memory usage.`
+That helps you to process a large number of objects without increasing memory usage.*
 
 When wrapped in a ``list(...)`` statement, it returns a list of Batches by loading them into the memory.
 
@@ -229,7 +240,7 @@ __ https://docs.scale.com/reference#batch-list
     counter = 0
     for batch in batches:
         counter += 1
-        print(f'Downloading batch {counter} | {batch.name} | {batch.project}')
+        print(f"Downloading batch {counter} | {batch.name} | {batch.project}")
 
     # Alternative for accessing as a Batch list
     batch_list = list(batches)
@@ -247,12 +258,16 @@ __ https://docs.scale.com/reference#project-creation
 
 .. code-block:: python
 
-    client.create_project(
-        project_name = 'test_project',
-        type = 'imageannotation,
-        params = {'instruction':'Please label the kittens'}
+    from scaleapi.tasks import TaskType
+
+    project = client.create_project(
+        project_name = "Test_Project",
+        task_type = TaskType.ImageAnnotation,
+        params = {"instruction": "Please label the kittens"},
     )
 
+    print(project.name)  # Test_Project
+
 Retrieve Project
 ^^^^^^^^^^^^^^^^
 
@@ -262,7 +277,7 @@ __ https://docs.scale.com/reference#project-retrieval
 
 .. code-block:: python
 
-    client.get_project(project_name = 'test_project')
+    client.get_project(project_name = "test_project")
 
 List Projects
 ^^^^^^^^^^^^^
@@ -290,9 +305,9 @@ __ https://docs.scale.com/reference#project-update-parameters
 .. code-block :: python
 
     data = client.update_project(
-        project_name='test_project',
+        project_name="test_project",
         patch = false,
-        instruction='update: Please label all the stuff',
+        instruction="update: Please label all the stuff",
     )
 
 Error handling
@@ -319,7 +334,7 @@ For example:
     from scaleapi.exceptions import ScaleException
 
     try:
-        client.create_task(TaskType.TextCollection, attachment='Some parameters are missing.')
+        client.create_task(TaskType.TextCollection, attachment="Some parameters are missing.")
     except ScaleException as err:
         print(err.code)  # 400
         print(err.message)  # Parameter is invalid, reason: "attachments" is required
diff --git a/docs/dev_requirements.txt b/docs/dev_requirements.txt
@@ -4,3 +4,4 @@ pre-commit==2.11.1
 isort>=5.7.0
 pytest>=6.2.2
 pylint>=2.7.2
+twine>=3.4.1
diff --git a/docs/developer_guide.md b/docs/developer_guide.md
@@ -19,7 +19,7 @@ $ pip install -r docs/dev_requirements.txt
 ```
 ### 3. Setup pre-commit
 
-Assure pre-commit<sup>1</sup> is installed:
+Assure pre-commit<sup>[1]</sup> is installed:
 ```bash
 $ pre-commit --version
 # pre-commit 2.11.1
@@ -52,6 +52,8 @@ Append following lines to the json file:
     },
 ```
 
+In Python SDK we follow [Google's Python Docstring Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for comments and docstring of modules, functions and classes. [Python Docstring Generator](https://marketplace.visualstudio.com/items?itemName=njpwerner.autodocstring) is a useful VS Code extension that helps to generate docstrings.
+
 ### 5. Running pre-commit Tests Manually
 
 You can run following command to run pre-commit linter for all files, without a commit. It provides a report for issues as well as fixes formatting.
diff --git a/publish.sh b/publish.sh
@@ -23,15 +23,7 @@ echo "Active Git Branch: ${BRANCH_NAME}" # release-1.0.5
 # BRANCH_PREFIX="${strarr[0]}"  # release
 # BRANCH_VERSION="${strarr[1]}" # 1.0.5
 
-while IFS= read -r line; do
-    if [[ $line == __version__* ]];
-    then
-        IFS=' = ' read -ra strarr <<< "$line"
-        PKG_VERSION=$( sed -e 's/^"//' -e 's/"$//' <<< "${strarr[1]}" )
-        echo "SDK Package Version: ${PKG_VERSION}"
-        break
-    fi
-done < "${DIR}/${VERSION_FILE}"
+PKG_VERSION=$(sed -n 's/^__version__ = //p' "${DIR}/${VERSION_FILE}" | sed -e 's/^"//' -e 's/"$//')
 
 if [ "$BRANCH_NAME" != "master" ];
 then
diff --git a/scaleapi/_version.py b/scaleapi/_version.py
@@ -1,2 +1,2 @@
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 __package_name__ = "scaleapi"
diff --git a/scaleapi/api.py b/scaleapi/api.py
@@ -12,7 +12,7 @@
 # Parameters for HTTP retry
 HTTP_TOTAL_RETRIES = 3  # Number of total retries
 HTTP_RETRY_BACKOFF_FACTOR = 2  # Wait 1, 2, 4 seconds between retries
-HTTP_STATUS_FORCE_LIST = [429, 504]  # Status codes to force retry
+HTTP_STATUS_FORCE_LIST = [429, 500, 504]  # Status codes to force retry
 HTTP_RETRY_ALLOWED_METHODS = frozenset({"GET", "POST"})
 
 
diff --git a/tests/test_client.py b/tests/test_client.py
@@ -19,8 +19,13 @@
 TEST_PROJECT_NAME = "scaleapi-python-sdk"
 
 try:
+    print(f"SDK Version: {scaleapi.__version__}")
     test_api_key = os.environ["SCALE_TEST_API_KEY"]
-    client = scaleapi.ScaleClient(test_api_key, "pytest")
+
+    if test_api_key.startswith("test_") or test_api_key.endswith("|test"):
+        client = scaleapi.ScaleClient(test_api_key, "pytest")
+    else:
+        raise Exception("Please provide a valid TEST environment key.")
 except KeyError as err:
     raise Exception(
         "Please set the environment variable SCALE_TEST_API_KEY to run tests."

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-__version__ = "2.0.0"`
	`1`	`+__version__ = "2.0.1"`
`2`	`2`	`__package_name__ = "scaleapi"`