Better PyPI scripting for releasing versions (#6)

oohlaf · web-flow · commit 1e85a28f570e · 2018-02-19T00:27:25.000+01:00
Any tag will create source and wheel distributions on Travis CI and AppVeyor, but only the v0.1.2 formatted tags will get a release on production PyPI. Any other tag, including v0.1.2.dev3, will be uploaded to Test PyPI.

Small fixes:
The brew installer under OSX now outputs to stderr.
This commit also includes some Flake8 fixes.
diff --git a/.appveyor/appveyor-bootstrap.py b/.appveyor/appveyor-bootstrap.py
@@ -1,8 +1,11 @@
 """
-AppVeyor will at least have few Pythons around so there's no point of implementing a bootstrapper in PowerShell.
+AppVeyor will at least have few Pythons around so there's no point of
+implementing a bootstrapper in PowerShell.
 
-This is a port of https://github.com/pypa/python-packaging-user-guide/blob/master/source/code/install.ps1
-with various fixes and improvements that just weren't feasible to implement in PowerShell.
+This is a port of
+https://github.com/pypa/python-packaging-user-guide/blob/master/source/code/install.ps1
+with various fixes and improvements that just weren't feasible to
+implement in PowerShell.
 """
 from __future__ import print_function
 
@@ -40,15 +43,20 @@
     ("3.6", "32"): BASE_URL + "3.6.0/python-3.6.0.exe",
 }
 INSTALL_CMD = {
-    # Commands are allowed to fail only if they are not the last command.  Eg: uninstall (/x) allowed to fail.
+    # Commands are allowed to fail only if they are not the last command.
+    # Eg: uninstall (/x) allowed to fail.
     "2.6": [["msiexec.exe", "/L*+!", "install.log", "/qn", "/x", "{path}"],
-            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}", "TARGETDIR={home}"]],
+            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}",
+             "TARGETDIR={home}"]],
     "2.7": [["msiexec.exe", "/L*+!", "install.log", "/qn", "/x", "{path}"],
-            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}", "TARGETDIR={home}"]],
+            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}",
+             "TARGETDIR={home}"]],
     "3.3": [["msiexec.exe", "/L*+!", "install.log", "/qn", "/x", "{path}"],
-            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}", "TARGETDIR={home}"]],
+            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}",
+             "TARGETDIR={home}"]],
     "3.4": [["msiexec.exe", "/L*+!", "install.log", "/qn", "/x", "{path}"],
-            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}", "TARGETDIR={home}"]],
+            ["msiexec.exe", "/L*+!", "install.log", "/qn", "/i", "{path}",
+             "TARGETDIR={home}"]],
     "3.5": [["{path}", "/quiet", "TargetDir={home}"]],
     "3.6": [["{path}", "/quiet", "TargetDir={home}"]],
 }
@@ -82,7 +90,7 @@ def install_python(version, arch, home):
         log.info("Running '%s'.", " ".join(cmd))
         try:
             check_call(cmd)
-        except Exception as exc:
+        except Exception:
             log.exception("Failed command '%s'.", " ".join(cmd))
             if exists("install.log"):
                 with open("install.log") as fh:
@@ -99,7 +107,7 @@ def download_python(version, arch):
     for _ in range(3):
         try:
             return download_file(URLS[version, arch], "installer.exe")
-        except Exception as exc:
+        except Exception:
             log.exception("Failed to download.")
         log.info("Retrying ...")
 
@@ -126,6 +134,9 @@ def install_packages(home, *packages):
     logging.basicConfig(level=logging.DEBUG,
                         format="%(message)s")
 
-    install_python(environ['PYTHON_VERSION'], environ['PYTHON_ARCH'], environ['PYTHON_HOME'])
+    install_python(environ['PYTHON_VERSION'],
+                   environ['PYTHON_ARCH'],
+                   environ['PYTHON_HOME'])
     install_pip(environ['PYTHON_HOME'])
-    install_packages(environ['PYTHON_HOME'], "setuptools>=36.4.0", "wheel")
+    install_packages(environ['PYTHON_HOME'],
+                     "setuptools>=36.4.0", "wheel")
diff --git a/.travis.yml b/.travis.yml
@@ -12,13 +12,15 @@ env:
     - secure: Od+iImlaOP2qzkGstfj3TTmhqWDdtZ1+CGFPJnHzZ9mj9l3Pg1Aek369f1MsGBXZeJqojiTx1xg/XzW3xH39QzbuynMGEFYOAZDnQMJsT990H1vKyHsoQ86t4n0T7FEIs6B5BzP1oDROdADHcxmRQ1BoSvWumbq/isGewArGO0M=
     # TWINE_PASSWORD
     - secure: cMN061O5PjB3B8+iSEgyo/qe3QPSDW8GaAupYtHKLnxT2M6JiepuVzZL40y9I1QTnZQMf8VIKH7XExLkgyVvbKZyQCctp9tioBBiAafp6uhAQMLSzj1+03gBzaJQJySPMLX4Snfa/TuhOqym5m4kYQTX/p3qCImzkgxWW1SV+rc=
-    - TWINE_REPOSITORY=https://test.pypi.org/legacy/
-    - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy/
+    - TWINE_TEST_REPOSITORY=pypitest
+    - TWINE_TEST_REPOSITORY_URL=https://test.pypi.org/legacy/
+    - TWINE_PROD_REPOSITORY=pypi
+    - TWINE_PROD_REPOSITORY_URL=https://upload.pypi.org/legacy/
 
 stages:
   - name: test
   - name: deploy
-    if: type = push AND tag =~ ^v\d+\.\d+\.\d+
+    if: type = push AND tag IS present
 
 os:
   - linux
@@ -39,23 +41,23 @@ python:
 before_install:
   - if [[ $TRAVIS_OS_NAME == "osx" ]]; then
       echo "Updating brew packages.";
-      brew update || travis_terminate 1;
+      brew update 1>&2 || travis_terminate 1;
       if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then
         echo "Installing python 2.";
-        brew upgrade python2 || travis_terminate 1;
+        brew upgrade python2 1>&2 || travis_terminate 1;
         virtualenv venv -p python2;
       elif [[ $TRAVIS_PYTHON_VERSION == "3.6" ]]; then
         echo "Installing python 3.";
-        brew install python3 || travis_terminate 1;
+        brew install python3 1>&2 || travis_terminate 1;
         virtualenv venv -p python3;
       fi;
       source venv/bin/activate || travis_terminate 1;
       python --version || travis_terminate 1;
     fi
-  - if [[ $TRAVIS_PULL_REQUEST == false && $TRAVIS_TAG =~ v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+  - if [[ $TRAVIS_PULL_REQUEST == false && -n $TRAVIS_TAG ]]; then
       if [[ $TRAVIS_OS_NAME == "osx" ]]; then
         echo "Installing pandoc.";
-        brew install pandoc || travis_terminate 1;
+        brew install pandoc 1>&2 || travis_terminate 1;
       fi;
       echo "Installing pypandoc.";
       pip install pypandoc || travis_terminate 1;
@@ -67,15 +69,15 @@ install:
 script: 
   - python setup.py test || travis_terminate 1
   # Need to pick a Python version to create the source distribution.
-  - if [[ $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == linux && $TRAVIS_PULL_REQUEST == false && $TRAVIS_TAG =~ v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+  - if [[ $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == linux && $TRAVIS_PULL_REQUEST == false && -n $TRAVIS_TAG ]]; then
       echo "Creating source distribution.";
       python setup.py sdist || travis_terminate 1;
     else
       echo "Skip creation of source distribution.";
     fi
   # Binary wheels for Linux that are too platform specific are not allowed to
   # be uploaded on PyPI. As we need to compile C code, we only create them on OSX.
-  - if [[ $TRAVIS_OS_NAME == osx && $TRAVIS_PULL_REQUEST == false && $TRAVIS_TAG =~ v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+  - if [[ $TRAVIS_OS_NAME == osx && $TRAVIS_PULL_REQUEST == false && -n $TRAVIS_TAG ]]; then
       echo "Creating binary distribution wheel.";
       python setup.py bdist_wheel || travis_terminate 1;
     else
@@ -124,9 +126,17 @@ jobs:
       script:
         - echo "Check build status."
         - python .travis/travis-release.py || travis_terminate 1
-        - echo "Deploy to PyPI stage."
+        - echo "Installing Twine."
         - pip install twine || travis_terminate 1
-        - twine upload dist/* --skip-existing || travis_terminate 1
+        - if [[ $TRAVIS_PULL_REQUEST == false && -n $TRAVIS_TAG ]]; then
+            if [[ $TRAVIS_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+              echo "Deploy to production PyPI.";
+              twine upload -r $TWINE_PROD_REPOSITORY --repository-url $TWINE_PROD_REPOSITORY_URL dist/* --skip-existing || travis_terminate 1;
+            else
+              echo "Deploy to test PyPI.";
+              twine upload -r $TWINE_TEST_REPOSITORY --repository-url $TWINE_TEST_REPOSITORY_URL dist/* --skip-existing || travis_terminate 1;
+            fi;
+          fi
 
 notifications:
   email:
diff --git a/.travis/travis-release.py b/.travis/travis-release.py
@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+import re
 import shutil
 import stat
 
@@ -77,7 +78,7 @@ def api_request(url, token):
                 request.add_header('Authorization', 'token {}'.format(token))
             response = urlopen(request)
             break
-        except Exception as exc:
+        except Exception:
             log.exception("Failed to call API.")
         log.info("Retrying ...")
         sleep(i)
@@ -240,18 +241,30 @@ def check_appveyor_tagged_build(url, tag, token):
 
 
 def check_code_version(filenames, tag):
-    """Check if the asset filenames relate to the tag version."""
-    wheel_part = "-{}-".format(tag[1:])
-    targz_part = "-{}.tar.gz".format(tag[1:])
+    """Check if the asset filenames relate to the tag version.
+
+    When tag is in the form v0.1.2 the first character is stripped.
+
+    Raises ReleaseVersionException when the tag is not found in any
+    of the asset file names.
+    """
+    if re.match(r"v\d+\.\d+\.\d+", tag) is not None:
+        file_tag = tag[1:]
+    else:
+        file_tag = tag
+    wheel_part = "-{}-".format(file_tag)
+    targz_part = "-{}.tar.gz".format(file_tag)
     for filename in filenames:
         if (filename[-4:] == '.whl') and (wheel_part not in filename):
-            log.info("Filename '%s' does not correspond "
-                     "to tag '%s'.", filename, tag)
-            return False
+            log.error("Filename '%s' does not correspond "
+                      "to tag '%s'.", filename, tag)
+            raise ReleaseVersionException("Version mismatch "
+                                          "between code and tag")
         if (filename[-7:] == '.tar.gz') and (targz_part not in filename):
-            log.info("Filename '%s' does not correspond "
-                     "to tag '%s'.", filename, tag)
-            return False
+            log.error("Filename '%s' does not correspond "
+                      "to tag '%s'.", filename, tag)
+            raise ReleaseVersionException("Version mismatch "
+                                          "between code and tag")
     return True
 
 
@@ -273,6 +286,4 @@ def check_code_version(filenames, tag):
                                                    gh_url, tag,
                                                    gh_token)
         log.info("All assets downloaded for tagged release '%s'.", tag)
-        if not check_code_version(filenames, tag):
-            raise ReleaseVersionException("Version mismatch "
-                                          "between code and tag")
+        check_code_version(filenames, tag)
diff --git a/appveyor.yml b/appveyor.yml
@@ -88,9 +88,7 @@ test_script:
 
 after_test:
   - ps: |
-      if ($env:APPVEYOR_REPO_TAG -and
-          $env:APPVEYOR_REPO_TAG_NAME -match "v\d\.\d\.\d")
-      {
+      if ($env:APPVEYOR_REPO_TAG -eq $true) {
           Write-Host "Installing pandoc."
           cinst --no-progress pandoc
           if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
diff --git a/setup.py b/setup.py
@@ -8,7 +8,7 @@
     from io import open
 
 
-VERSION = '1.0.0.dev2'
+VERSION = '1.0.0.dev3'
 GITHUB_URL = 'https://github.com/oohlaf/python-whirlpool'
 DOWNLOAD_URL = '{}/archive/{}.zip'.format(GITHUB_URL, VERSION)
 
