Follow up PR to the previous testing fixes (#1847)

* Follow up PR to the previous testing fixes

* Force tox v4

* Try adding an echo for debugging

* Debug extras,sso

* Escape commas?

* Bash brace expansion doesn't work the way you thought it does

* Install tox4 in docker files

* Use --installpkg for tox4; does it work with wildcards?

* Expand --installpkg wheels

* Remove unnecessary factoring

Author: sfc-gh-bwarsaw

.github/workflows/build_test.yml

@@ -36,15 +36,15 @@ jobs:
       - name: Upgrade setuptools, pip and wheel
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
-        run: python -m pip install tox
+        run: python -m pip install tox>=4
       - name: Set PY
         run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - name: Run fix_lint
-        run: python -m tox -e fix_lint
+        run: python -m tox run -e fix_lint
 
   dependency:
     name: Check dependency
@@ -61,9 +61,9 @@ jobs:
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
       - name: Install tox
-        run: python -m pip install tox tox-external-wheels
+        run: python -m pip install tox>=4
       - name: Run tests
-        run: python -m tox -e dependency
+        run: python -m tox run -e dependency
 
   build:
     needs: lint
@@ -152,17 +152,17 @@ jobs:
       - name: Upgrade setuptools, pip and wheel
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
-        run: python -m pip install tox tox-external-wheels
+        run: python -m pip install tox>=4
       - name: Run tests
-        run: python -m tox -e "py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci"
+        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'`
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
       - name: Combine coverages
-        run: python -m tox -e coverage --skip-missing-interpreters false
+        run: python -m tox run -e coverage --skip-missing-interpreters false
         shell: bash
       - uses: actions/upload-artifact@v3
         with:
@@ -201,9 +201,9 @@ jobs:
       - name: Upgrade setuptools, pip and wheel
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
-        run: python -m pip install tox
+        run: python -m pip install tox>=4
       - name: Run tests
-        run: python -m tox -e olddriver
+        run: python -m tox run -e olddriver
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
@@ -233,9 +233,9 @@ jobs:
       - name: Upgrade setuptools, pip and wheel
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
-        run: python -m pip install tox
+        run: python -m pip install tox>=4
       - name: Run tests
-        run: python -m tox -e noarrowextension
+        run: python -m tox run -e noarrowextension
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
@@ -347,7 +347,7 @@ jobs:
       - name: Upgrade setuptools and pip
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
-        run: python -m pip install tox
+        run: python -m pip install tox>=4
       - name: Collect all coverages to one dir
         run: |
           python -c '
@@ -362,7 +362,7 @@ jobs:
               print("{} copy to {}".format(src_file, dst_file))
               shutil.copy(str(src_file), str(dst_file))'
       - name: Combine coverages
-        run: python -m tox -e coverage
+        run: python -m tox run -e coverage
       - name: Publish html coverage
         uses: actions/upload-artifact@v3
         with:

ci/docker/connector_test_lambda/Dockerfile310

@@ -7,6 +7,6 @@ RUN chmod 777 /home/user/snowflake-connector-python
 ENV PATH="${PATH}:/opt/python/cp310-cp310/bin/"
 ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
 
-RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+RUN pip3 install -U pip setuptools wheel tox>=4
 
 CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile311

@@ -7,6 +7,6 @@ RUN chmod 777 /home/user/snowflake-connector-python
 ENV PATH="${PATH}:/opt/python/cp311-cp311/bin/"
 ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
 
-RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+RUN pip3 install -U pip setuptools wheel tox>=4
 
 CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile312

@@ -5,6 +5,6 @@ RUN chmod 777 /home/user/snowflake-connector-python
 ENV PATH="${PATH}:/opt/python/cp312-cp312/bin/"
 ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
 
-RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+RUN pip3 install -U pip setuptools wheel tox>=4
 
 CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile38

@@ -7,6 +7,6 @@ RUN chmod 777 /home/user/snowflake-connector-python
 ENV PATH="${PATH}:/opt/python/cp38-cp38/bin/"
 ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
 
-RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+RUN pip3 install -U pip setuptools wheel tox>=4
 
 CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile39

@@ -7,6 +7,6 @@ RUN chmod 777 /home/user/snowflake-connector-python
 ENV PATH="${PATH}:/opt/python/cp39-cp39/bin/"
 ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
 
-RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+RUN pip3 install -U pip setuptools wheel tox>=4
 
 CMD [ "app.handler" ]

ci/docker/connector_test_lambda/app.py

@@ -1,6 +1,7 @@
 import logging
 import sys
 import xml.etree.ElementTree as ET
+from glob import glob
 from pathlib import Path
 from subprocess import PIPE, Popen
 
@@ -13,23 +14,30 @@
 def run_tests():
     """Run tests using tox"""
     LOGGER.info("Running tests..")
+    # Get the list of wheels; pass them to tox, one per --installpkg option.
+    args = [
+        "python",
+        "-m",
+        "tox",
+        "run",
+        "-e",
+        f"py{PY_SHORT_VER}-lambda-ci",
+        "-c",
+        f"{REPO_PATH}/tox.ini",
+        "--workdir",
+        REPO_PATH,
+    ]
+    for wheel in glob(f"{REPO_PATH}/dist/*.whl"):
+        args.extend(["--installpkg", wheel])
+
+    LOGGER.info(f"Popen args: {args}")
+
     test_log, err = Popen(
-        [
-            "python",
-            "-m",
-            "tox",
-            "-e",
-            f"py{PY_SHORT_VER}{{-lambda}}-ci",
-            "-c",
-            f"{REPO_PATH}/tox.ini",
-            "--workdir",
-            REPO_PATH,
-            "--external_wheels",
-            f"{REPO_PATH}/dist/*.whl",
-        ],
+        args,
         stdout=PIPE,
         stderr=PIPE,
     ).communicate()
+
     LOGGER.info(test_log)
     LOGGER.info(err)
     return test_log.decode("utf-8")

test/README.md

@@ -25,21 +25,22 @@ tox -e "fix_lint,py37{,-pandas,-sso}"
 ```
 
 **NOTE** Some integration tests may be sensitive to the cloud provider of the
-account that the test suite connects to.  By default, when testing locally,
-the "provider" is called `dev`, but you may see some test failures with this
-provider (which isn't really a provider).  To eliminate this false failure,
-set the environment variable `cloud_provider` to one of `aws`, `gcp`, or
-`azure` as appropriate for the account you're running integration tests
-against.  This is handled correctly in CI so should only affect local
-developers.  In the future, we'll try to make this automatic by querying the
-account after the connection is made.
+account that the test suite connects to.  The default `dev` provider acts like
+all cloud providers, but you may see some test failures during integration
+tests, depending on the actual cloud of the account you are connecting to.  To
+eliminate any such false failure, set the environment variable
+`cloud_provider` to one of `aws`, `gcp`, or `azure` as appropriate for the
+account you're running the integration tests against.  This is handled
+correctly in CI so should only affect your local testing.  In the future,
+we'll try to make this automatic by querying the account after the connection
+is made.
 
 ### Running a single test
 
 Enter the tox environment you want (e.g. `py38`) and run `pytest` from there:
 
 ```shell
-source .tox/py38/bin/activate
+. .tox/py38/bin/activate
 pytest -v test/integ/test_connection.py::test_basic
 ```
 

tox.ini

@@ -16,7 +16,7 @@ source = src/snowflake/connector
          */fips_env/lib/python*/site-packages/snowflake/connector
 
 [tox]
-minversion = 3.7
+minversion = 4
 envlist = fix_lint,
           py{37,38,39,310,311,312}-{extras,unit-parallel,integ,pandas,sso},
           coverage