mraba/rt-exp: rt parallelisation

Author: sfc-gh-mraba

ci/test_docker.sh

@@ -7,11 +7,11 @@
 set -o pipefail
 
 # In case this is ran from dev-vm
-PYTHON_ENV=${1:-3.7}
+PYTHON_ENV=${1:-3.8}
 
 # Set constants
-THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-SQLALCHEMY_DIR="$( dirname "${THIS_DIR}")"
+THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SQLALCHEMY_DIR="$(dirname "${THIS_DIR}")"
 WORKSPACE=${WORKSPACE:-$SQLALCHEMY_DIR}
 source $THIS_DIR/set_base_image.sh
 
@@ -35,19 +35,19 @@ echo "[Info] Start building docker image and testing"
 
 user_id=$(id -u ${USER})
 docker run \
-    --rm \
-    --network=host \
-    -e TERM=vt102 \
-    -e PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    -e OPENSSL_FIPS=1 \
-    -e LOCAL_USER_ID=${user_id} \
-    -e AWS_ACCESS_KEY_ID \
-    -e AWS_SECRET_ACCESS_KEY \
-    -e SF_REGRESS_LOGS \
-    -e SF_PROJECT_ROOT \
-    -e cloud_provider \
-    -e JENKINS_HOME \
-    -e GITHUB_ACTIONS \
-    --mount type=bind,source="${SQLALCHEMY_DIR}",target=/home/user/snowflake-sqlalchemy \
-    $(docker build --pull --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg GOSU_URL="$GOSU_URL" -q .) \
-    /home/user/snowflake-sqlalchemy/ci/test_linux.sh ${PYTHON_ENV}
+  --rm \
+  --network=host \
+  -e TERM=vt102 \
+  -e PIP_DISABLE_PIP_VERSION_CHECK=1 \
+  -e OPENSSL_FIPS=1 \
+  -e LOCAL_USER_ID=${user_id} \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e SF_REGRESS_LOGS \
+  -e SF_PROJECT_ROOT \
+  -e cloud_provider \
+  -e JENKINS_HOME \
+  -e GITHUB_ACTIONS \
+  --mount type=bind,source="${SQLALCHEMY_DIR}",target=/home/user/snowflake-sqlalchemy \
+  $(docker build --pull --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg GOSU_URL="$GOSU_URL" -q .) \
+  /home/user/snowflake-sqlalchemy/ci/test_linux.sh ${PYTHON_ENV}

ci/test_linux.sh

@@ -6,7 +6,7 @@
 #   - This script assumes that ../dist/repaired_wheels has the wheel(s) built for all versions to be tested
 #   - This is the script that test_docker.sh runs inside of the docker container
 
-PYTHON_VERSIONS="${1:-3.8 3.9 3.10 3.11}"
+PYTHON_VERSIONS="${1:-3.8 3.9 3.10 3.11 3.12 3.13 3.14}"
 THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SQLALCHEMY_DIR="$(dirname "${THIS_DIR}")"
 
@@ -19,7 +19,7 @@ for PYTHON_VERSION in ${PYTHON_VERSIONS}; do
   echo "[Info] Testing with ${PYTHON_VERSION}"
   SHORT_VERSION=$(python3 -c "print('${PYTHON_VERSION}'.replace('.', ''))")
   SQLALCHEMY_WHL=$(ls $SQLALCHEMY_DIR/dist/snowflake_sqlalchemy-*-py3-none-any.whl | sort -r | head -n 1)
-  TEST_ENVLIST=fix_lint,py${SHORT_VERSION}-ci,py${SHORT_VERSION}-coverage
+  TEST_ENVLIST=fix_lint,py${SHORT_VERSION}-ci,py${SHORT_VERSION}-coverage,py${SHORT_VERSION}-pandas-ci,py${SHORT_VERSION}-pandas-coverage
   echo "[Info] Running tox for ${TEST_ENVLIST}"
   python3 -m tox -e ${TEST_ENVLIST} --installpkg ${SQLALCHEMY_WHL}
 done

tests/conftest.py

@@ -59,7 +59,6 @@ def pytest_addoption(parser):
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--ignore_v20_test"):
-        # --ignore_v20_test given in cli: skip sqlalchemy 2.0 tests
         skip_feature_v2 = pytest.mark.skip(
             reason="need remove --ignore_v20_test option to run"
         )
@@ -68,35 +67,6 @@ def pytest_collection_modifyitems(config, items):
                 item.add_marker(skip_feature_v2)
 
 
-@pytest.fixture(scope="session")
-def on_travis():
-    return os.getenv("TRAVIS", "").lower() == "true"
-
-
-@pytest.fixture(scope="session")
-def on_appveyor():
-    return os.getenv("APPVEYOR", "").lower() == "true"
-
-
-@pytest.fixture(scope="session")
-def on_public_ci(on_travis, on_appveyor):
-    return on_travis or on_appveyor
-
-
-def help():
-    print(
-        """Connection parameter must be specified in parameters.py,
-    for example:
-CONNECTION_PARAMETERS = {
-    'account': 'testaccount',
-    'user': 'user1',
-    'password': 'test',
-    'database': 'testdb',
-    'schema': 'public',
-}"""
-    )
-
-
 logger = getLogger(__name__)
 
 TZ_ENV_VAR: Literal["TZ"] = "TZ"
@@ -137,6 +107,11 @@ def external_stage():
         raise ValueError("External_stage is not set")
 
 
+@pytest.fixture(scope="session")
+def on_public_ci():
+    return running_on_public_ci()
+
+
 @pytest.fixture(scope="function")
 def base_location(external_stage, engine_testaccount):
     unique_id = str(uuid.uuid4())
@@ -325,6 +300,7 @@ def running_on_public_ci() -> bool:
 
 def pytest_runtest_setup(item) -> None:
     """Ran before calling each test, used to decide whether a test should be skipped."""
+    _ensure_optional_dependencies(item)
     test_tags = [mark.name for mark in item.iter_markers()]
 
     # Get what cloud providers the test is marked for if any
@@ -342,3 +318,9 @@ def pytest_runtest_setup(item) -> None:
         pytest.skip("cannot run this test on external CI")
     elif INTERNAL_SKIP_TAGS.intersection(test_tags) and not running_on_public_ci():
         pytest.skip("cannot run this test on internal CI")
+
+
+def _ensure_optional_dependencies(item):
+    """Skip optional-dependency tests when the dependency is unavailable."""
+    if "pandas" in item.keywords:
+        pytest.importorskip("pandas")

tests/test_core.py

@@ -171,7 +171,6 @@ def test_connect_args():
 
 
 def test_boolean_query_argument_parsing():
-
     engine = create_engine(
         URL(
             **CONNECTION_PARAMETERS,
@@ -1385,12 +1384,12 @@ def test_special_schema_character(db_parameters, on_public_ci):
     with connect(**options) as sf_conn:
         sf_connection = (
             sf_conn.cursor()
-            .execute("select current_database(), " "current_schema();")
+            .execute("select current_database(), current_schema();")
             .fetchall()
         )
     with create_engine(URL(**options)).connect() as sa_conn:
         sa_connection = sa_conn.execute(
-            text("select current_database(), " "current_schema();")
+            text("select current_database(), current_schema();")
         ).fetchall()
     # Teardown
     with connect(**options) as conn:
@@ -1823,6 +1822,7 @@ def test_normalize_and_denormalize_empty_string_column_name(engine_testaccount):
         )
 
 
+@pytest.mark.pandas
 def test_snowflake_sqlalchemy_as_valid_client_type():
     engine = create_engine(
         URL(**CONNECTION_PARAMETERS),

tests/test_pandas.py

@@ -10,8 +10,10 @@
 import uuid
 
 import numpy as np
-import pandas as pd
 import pytest
+
+pytest.importorskip("pandas")
+import pandas as pd
 import sqlalchemy
 from sqlalchemy import (
     Column,

tests/test_qmark.py

@@ -5,8 +5,10 @@
 import os
 import sys
 
-import pandas as pd
 import pytest
+
+pytest.importorskip("pandas")
+import pandas as pd
 from sqlalchemy import text
 
 THIS_DIR = os.path.dirname(os.path.realpath(__file__))

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 min_version = 4.0.0
 envlist = fix_lint,
-          py{37,38,39,310,311,312,313,314}{,-pandas},
+          py{38,39,310,311,312,313,314}{,-pandas},
           coverage,
 skip_missing_interpreters = true
 
@@ -10,9 +10,8 @@ package = external
 description = run the tests with pytest under {basepython}
 extras =
     development
-    pandas
+    pandas: pandas
 external_wheels =
-    py37-ci: dist/*.whl
     py38-ci: dist/*.whl
     py39-ci: dist/.whl
     py310-ci: dist/.whl
@@ -21,6 +20,7 @@ external_wheels =
     py313-ci: dist/.whl
     py314-ci: dist/.whl
 deps = pip
+  pytest-xdist
 passenv =
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
@@ -44,10 +44,12 @@ commands = pytest \
            --cov "snowflake.sqlalchemy" \
            --junitxml {toxworkdir}/junit_{envname}.xml \
            --ignore=tests/sqlalchemy_test_suite \
+           -n8 \
            {posargs:tests}
            pytest {env:SNOWFLAKE_PYTEST_OPTS:} \
            --cov "snowflake.sqlalchemy" --cov-append \
            --junitxml {toxworkdir}/junit_{envname}.xml \
+           -n8 \
            {posargs:tests/sqlalchemy_test_suite}
 
 [testenv:.pkg_external]
@@ -70,7 +72,7 @@ commands = coverage combine
            coverage xml -o {toxworkdir}/coverage.xml
            coverage html -d {toxworkdir}/htmlcov
            ;diff-cover --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}/coverage.xml
-depends = py37, py38, py39, py310, py311, py312, py313, py314
+depends = py38, py39, py310, py311, py312, py313, py314
 
 [testenv:fix_lint]
 description = format the code base to adhere to our styles, and complain about what we cannot do automatically