run integ tests only when secrets are available

Author: danabens

.github/workflows/build_release.yml

@@ -11,42 +11,50 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 3
-      matrix:
-        python-version: [3.5, 3.6, 3.7]
     steps:
-      - name: Check Environment
-        run: |
-          if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY  ]]; then
-            echo "Missing required environment variables."
-            echo "If this a pull request check your fork's action history."
-            exit 1
-          fi
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - uses: actions/checkout@v1
-      - name: Setup Python ${{matrix.python-version}}
+      - name: Setup Python 3.5
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.5
+      - name: Setup Python 3.6
         uses: actions/setup-python@v1
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.6
+      - name: Setup Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
       - name: Install Dependencies
-        run: pip install tox tox-gh-actions
+        run: pip install tox
       - name: Flake8
         run: tox -e flake8
       - name: Black Check
         run: tox -e black-check
       - name: Pylint
         run: tox -e pylint
-      # starts 3 jobs, 1 for each python version
-      - name: Test
-        run: tox
+      # runs unit tests for each python version
+      - name: Unit Tests
+        run: tox -- tests/unit
+        env:
+          AWS_DEFAULT_REGION: us-west-2
+      - name: Integration Tests
+      # pull requests are untrusted and do not have access to secrets needed for integ tests
+        if: github.event_name != 'pull_request'
+        run: tox -e py36 -- tests/integ
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: us-west-2
-          CODECOV_UPLOAD_TOKEN: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
+          COVERAGE_FILE: .coverage.integ
+          IGNORE_COVERAGE: '-'
+      - name: Upload Code Coverage
+        if: github.event_name == 'release'
+        run: |
+          coverage combine .coverage*
+          codecov -t {env:CODECOV_UPLOAD_TOKEN}
+        env:
+          CODECOV_UPLOAD_TOKEN: ${{ secrets.AWS_ACCESS_KEY_ID }}
 
   release:
     needs: build

.gitignore

@@ -28,6 +28,7 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 .tox/
 .coverage
+.coverage.integ
 .cache
 nosetests.xml
 coverage.xml

docs/conf.py

@@ -16,13 +16,13 @@
 import os
 import sys
 
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
 
 # -- Project information -----------------------------------------------------
 
-project = 'sagemaker-experiments'
+project = "sagemaker-experiments"
 copyright = u"%s, Amazon" % datetime.now().year
-author = 'Amazon Web Services'
+author = "Amazon Web Services"
 version = pkg_resources.require(project)[0].version
 
 # -- General configuration ---------------------------------------------------
@@ -42,7 +42,7 @@
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 source_suffix = ".rst"  # The suffix of source filenames.
 master_doc = "index"  # The master toctree document.
@@ -59,7 +59,7 @@
 
 pygments_style = "default"
 autoclass_content = "both"
-#autodoc_default_options = {"show-inheritance", "members", "undoc-members"}
+# autodoc_default_options = {"show-inheritance", "members", "undoc-members"}
 autodoc_member_order = "bysource"
 if "READTHEDOCS" in os.environ:
     html_theme = "default"
@@ -75,4 +75,4 @@
 autosummary_generate = True
 
 # autosectionlabel
-autosectionlabel_prefix_document = True
+autosectionlabel_prefix_document = True

tests/unit/test_tracker.py

@@ -196,6 +196,28 @@ def test_log_metric(under_test):
     under_test._metrics_writer.log_metric.assert_called_with("foo", 1.0, 1, now)
 
 
+def test_log_metric_attribute_error(under_test):
+    now = datetime.datetime.now()
+
+    exception = AttributeError
+
+    under_test._metrics_writer.log_metric.side_effect = exception
+
+    with pytest.raises(AttributeError):
+        under_test.log_metric("foo", 1.0, 1, now)
+
+
+def test_log_metric_attribute_error_warned(under_test):
+    now = datetime.datetime.now()
+
+    under_test._metrics_writer = None
+    under_test._warned_on_metrics = None
+
+    under_test.log_metric("foo", 1.0, 1, now)
+
+    assert under_test._warned_on_metrics == True
+
+
 def test_log_artifact(under_test):
     under_test.log_artifact("foo.txt", "name", "whizz/bang")
     under_test._artifact_uploader.upload_artifact.assert_called_with("foo.txt")

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{35,36,37},flake8,black-format,black-check,pylint
+envlist = py{35,36,37}
 
 skip_missing_interpreters = False
 ignore_basepython_conflict = True
@@ -50,8 +50,7 @@ passenv =
     AWS_DEFAULT_REGION
     SAGEMAKER_ENDPOINT
     CODECOV_UPLOAD_TOKEN
-    # used by codecov
-    GITHUB_*
+    COVERAGE_FILE
 whitelist_externals =
     echo
     codecov
@@ -61,7 +60,6 @@ commands =
     echo running env: {envname}
     coverage run --source smexperiments -m pytest {posargs} -m "not slow"
     {env:IGNORE_COVERAGE:} coverage report  --fail-under=95
-    codecov -e {envname} -t {env:CODECOV_UPLOAD_TOKEN}
 extras = test
 deps =
     boto3 >= 1.10.32
@@ -80,12 +78,6 @@ deps =
     flake8-future-import
 commands = flake8
 
-[gh-actions]
-python =
-    3.5: py35
-    3.6: py36
-    3.7: py37
-
 [testenv:black-format]
 # Used during development (before committing) to format .py files.
 deps = black
@@ -114,7 +106,6 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
     SAGEMAKER_ENDPOINT
-
 # {posargs} can be passed in by additional arguments specified when invoking tox.
 # Can be used to specify which tests to run, e.g.: tox -- -s
 commands =
@@ -138,4 +129,4 @@ deps =
     sphinx-rtd-theme
     readthedocs-sphinx-ext
 commands =
-    sphinx-build -T -W -b html -d _build/doctrees-readthedocs -D language=en . _build/html
+    sphinx-build -T -W -b html -d _build/doctrees-readthedocs -D language=en . _build/html