Tests: Clean up the Python version unit tests (#1092)

edmorley · web-flow · commit a98ef91566e5 · 2020-10-07T15:10:20.000+01:00
* Fixes the "Installing &lt;version&gt;" assertions so that they don't false
  positive against the "please upgrade to &lt;version&gt;" output.
* Removes modification of test fixtures during tests, since it can lead
  to failures depending on test order, and confusion when debugging.
* Updates the PyPy version warning tests to use a slightly newer (but
  still not latest) PyPy version, which means that the test now passes
  on Cedar-14 and can be unskipped.
* Switches to using an empty requirements file for version tests that
  duplicate the main test, to save spending time installing dependencies
  unnecessarily.
* Switches the NLTK test to using the default buildpack Python version,
  rather than an ancient Python 3.6.
* Skips the Python 3.7/3.8 tests on Cedar-14 rather than asserting
  failure, since we know they'll never pass due to Cedar-14's libssl being
  older than required.
* Removes redundant `testSqliteInstall` test since it duplicates the
  Python version install tests.

Longer term I'll be moving many of the unit tests to Hatchet, however
this at least makes the tests more dependable in the meantime.

Closes @W-8060219@.
Closes @W-8176779@.

[skip changelog]
diff --git a/test/fixtures/nltk/runtime.txt b/test/fixtures/nltk/runtime.txt
diff --git a/test/fixtures/pypy2_7_warn/runtime.txt b/test/fixtures/pypy2_7_warn/runtime.txt
@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.1
diff --git a/test/fixtures/pypy3_6_warn/runtime.txt b/test/fixtures/pypy3_6_warn/runtime.txt
@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.1
diff --git a/test/fixtures/python2_warn/requirements.txt b/test/fixtures/python2_warn/requirements.txt
@@ -1 +0,0 @@
-requests
diff --git a/test/fixtures/python3_4_warn/requirements.txt b/test/fixtures/python3_4_warn/requirements.txt
@@ -1 +0,0 @@
-flask
diff --git a/test/fixtures/python3_5_warn/requirements.txt b/test/fixtures/python3_5_warn/requirements.txt
@@ -1 +0,0 @@
-flask
diff --git a/test/fixtures/python3_6/runtime.txt b/test/fixtures/python3_6/runtime.txt
@@ -1 +1 @@
-python-3.6.8
+python-3.6.12
diff --git a/test/fixtures/python3_6_warn/requirements.txt b/test/fixtures/python3_6_warn/requirements.txt
@@ -1 +0,0 @@
-requests
diff --git a/test/fixtures/python3_7/runtime.txt b/test/fixtures/python3_7/runtime.txt
@@ -1 +1 @@
-python-3.7.2
+python-3.7.9
diff --git a/test/fixtures/python3_7_warn/requirements.txt b/test/fixtures/python3_7_warn/requirements.txt
@@ -1 +0,0 @@
-requests
diff --git a/test/fixtures/python3_8/runtime.txt b/test/fixtures/python3_8/runtime.txt
@@ -1 +1 @@
-python-3.8.2
+python-3.8.6
diff --git a/test/fixtures/python3_8_warn/requirements.txt b/test/fixtures/python3_8_warn/requirements.txt
@@ -1 +0,0 @@
-requests
diff --git a/test/fixtures/pythonDefault/requirements.txt b/test/fixtures/pythonDefault/requirements.txt
diff --git a/test/fixtures/python_version_unspecified/requirements.txt b/test/fixtures/python_version_unspecified/requirements.txt
diff --git a/test/run-deps b/test/run-deps
@@ -60,12 +60,6 @@ testPysqlite() {
   assertCapturedSuccess
 }
 
-testSqliteInstall() {
-  compile "pythonDefault"
-  assertNotCaptured "Sqlite3 failed to install."
-  assertCapturedSuccess
-}
-
 testCffi() {
   compile "cffi"
   assertCaptured "cffi"
diff --git a/test/run-features b/test/run-features
@@ -59,22 +59,23 @@ testPipenvLock() {
   assertCapturedSuccessWithStdErr
 }
 
-testPipenvVersion() {
+testPipenvPythonVersion3_6() {
   compile "pipenv-version"
-  assertCaptured $DEFAULT_PYTHON_VERSION
+  assertCaptured "Installing ${LATEST_36}"
   # Can't use `assertCapturedSuccess` since stderr contains:
   # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
   assertCapturedSuccessWithStdErr
 }
 
-testPipenvVersion2() {
+testPipenvPythonVersion2_7() {
   compile "pipenv-version2"
-  assertCaptured $LATEST_27
+  assertCaptured "Installing ${LATEST_27}"
   # Can't use `assertCapturedSuccess` since stderr contains:
   # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
   assertCapturedSuccessWithStdErr
 }
-testPipenvFullVersion() {
+
+testPipenvPythonFullVersion() {
   compile "pipenv-full-version"
   assertCaptured "3.6.3"
   # Can't use `assertCapturedSuccess` since stderr contains:
diff --git a/test/run-versions b/test/run-versions
@@ -4,59 +4,58 @@
 # shellcheck source=bin/default_pythons
 source "bin/default_pythons"
 
-testPythonDefault() {
-  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
-  compile "pythonDefault"
-  assertCaptured $DEFAULT_PYTHON_VERSION
+testPythonVersionUnspecified() {
+  compile "python_version_unspecified"
+  assertCaptured "Installing ${DEFAULT_PYTHON_VERSION}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
 
-testPython2() {
-    updateVersion "python2" $LATEST_27
-    echo $LATEST_27 > "runtime.txt"
-    compile "python2"
-    assertCaptured $LATEST_27
-    assertCaptured "python-2-7-eol-faq";
-    assertNotCaptured "security update"
-    assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7() {
+  compile "python2"
+  assertCaptured "Installing ${LATEST_27}"
+  assertCaptured "python-2-7-eol-faq";
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
-testPython2_warn() {
-    compile "python2_warn"
-    assertCaptured "python-2.7.15"
-    assertCaptured "python-2-7-eol-faq";
-    assertCaptured "Only the latest version"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7_warn() {
+  compile "python2_warn"
+  assertCaptured "Installing python-2.7.15"
+  assertCaptured "python-2-7-eol-faq";
+  assertCaptured "Only the latest version"
+  assertCaptured "${LATEST_27}"
+  assertCapturedSuccess
 }
 
 testPython3_4() {
   compile "python3_4"
-  assertCaptured $LATEST_34
+  assertCaptured "Installing ${LATEST_34}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
+  assertCaptured "Installing SQLite3"
   # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
   # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
   assertCapturedSuccessWithStdErr
 }
 
 testPython3_4_warn() {
   compile "python3_4_warn"
-  assertCaptured "python-3.4.9"
+  assertCaptured "Installing python-3.4.9"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_34}"
   # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
   # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
   assertCapturedSuccessWithStdErr
 }
 
 testPython3_5() {
   compile "python3_5"
-  assertCaptured $LATEST_35
+  assertCaptured "Installing ${LATEST_35}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
@@ -65,15 +64,15 @@ testPython3_5() {
 
 testPython3_5_warn() {
   compile "python3_5_warn"
-  assertCaptured "python-3.5.6"
+  assertCaptured "Installing python-3.5.6"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_35}"
   assertCapturedSuccess
 }
 
 testPython3_6() {
-  updateVersion "python3_6" $LATEST_36
   compile "python3_6"
-  assertCaptured $LATEST_36
+  assertCaptured "Installing ${LATEST_36}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
@@ -82,62 +81,60 @@ testPython3_6() {
 
 testPython3_6_warn() {
   compile "python3_6_warn"
-  assertCaptured "python-3.6.7"
+  assertCaptured "Installing python-3.6.7"
   assertCaptured "security update!"
-  assertCaptured "Installing SQLite3"
+  assertCaptured "${LATEST_36}"
   assertCapturedSuccess
 }
 
 testPython3_7() {
-  updateVersion "python3_7" $LATEST_37
-  compile "python3_7"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_37
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_7"
+  assertCaptured "Installing ${LATEST_37}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
 testPython3_7_warn() {
-  compile "python3_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.7.1"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_7_warn"
+  assertCaptured "Installing python-3.7.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_37}"
+  assertCapturedSuccess
 }
 
 testPython3_8() {
-  updateVersion "python3_8" $LATEST_38
-  compile "python3_8"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_38
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_8"
+  assertCaptured "Installing ${LATEST_38}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
 testPython3_8_warn() {
-  compile "python3_8_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.8.0"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_8_warn"
+  assertCaptured "Installing python-3.8.0"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_38}"
+  assertCapturedSuccess
 }
 
 testPython3_9() {
@@ -146,8 +143,8 @@ testPython3_9() {
     return
   fi
   compile "python3_9"
+  assertCaptured "Installing ${LATEST_39}"
   assertNotCaptured "security update"
-  assertCaptured $LATEST_39
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
@@ -163,8 +160,9 @@ testPython3_9_warn() {
     return
   fi
   compile "python3_9_warn"
-  assertCaptured "python-3.9.0"
+  assertCaptured "Installing python-3.9.0"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_39}"
   assertCapturedSuccess
 }
 
@@ -177,53 +175,44 @@ testPythonVersionInvalid() {
 
 testPypy3_6() {
   compile "pypy3_6"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_36}"
   assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_36"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
 testPypy3_6_warn() {
   compile "pypy3_6_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_36"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy3.6-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_36}"
+  assertCapturedSuccess
 }
 
 testPypy2_7() {
   compile "pypy2_7"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_27}"
   assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_27"
   assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
 testPypy2_7_warn() {
   compile "pypy2_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_27"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy2.7-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_27}"
+  assertCapturedSuccess
 }
 
 testStickyPythonVersion() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
   assertCaptured "Installing python-3.6.7"
   assertCapturedSuccess
-  compile "no-runtime-txt" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  compile "python_version_unspecified" "$cache_dir"
+  assertNotCaptured "Installing python"
+  assertCaptured "security update!"
   assertCapturedSuccess
   # Whilst this file seems like an implementation detail (so something that should
   # not be tested), we must guarantee the filename remains consistent for backwards
diff --git a/test/utils b/test/utils
@@ -56,11 +56,6 @@ resetCapture()
   unset rtrn # deprecated
 }
 
-updateVersion()
-{
-  echo "$2" > "test/fixtures/${1}/runtime.txt"
-}
-
 assertCapturedEquals()
 {
   assertEquals "$@" "$(cat ${STD_OUT})"
