change: update scikit-learn, scipy, and apache-airflow deps for dependabot (#4103)

Author: benieric

.githooks/pre-push

@@ -1,5 +1,5 @@
 #!/bin/sh
-# this pre-push hook runs style checks and unit tests in python 3.6, 3.7, and 3.8 using tox.
+# this pre-push hook runs style checks and unit tests in python 3.8, 3.9, and 3.10 using tox.
 
 set -e
 

requirements/extras/scipy_requirements.txt

@@ -1 +1 @@
-scipy==1.7.3
+scipy==1.10.1

requirements/extras/test_requirements.txt

@@ -12,7 +12,7 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.6.3
+apache-airflow==2.7.0
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0
@@ -21,9 +21,9 @@ sagemaker-experiments==0.1.35
 Jinja2==3.0.3
 pyvis==0.2.1
 pandas>=1.3.5,<1.5
-scikit-learn==1.0.2
+scikit-learn==1.3.0
 cloudpickle==2.2.1
-scipy==1.7.3
+scipy==1.10.1
 urllib3>=1.26.8,<1.26.15
 docker>=5.0.2,<7.0.0
 PyYAML==6.0

setup.py

@@ -92,7 +92,7 @@ def read_requirements(filename):
     url="https://github.com/aws/sagemaker-python-sdk/",
     license="Apache License 2.0",
     keywords="ML Amazon AWS AI Tensorflow MXNet",
-    python_requires=">= 3.7",
+    python_requires=">= 3.8",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",

tests/data/remote_function/requirements.txt

@@ -1 +1 @@
-scipy==1.7.3
+scipy==1.10.1

tests/integ/sagemaker/experiments/test_run.py

@@ -693,7 +693,7 @@ def _generate_estimator(
         sagemaker_client_config=sagemaker_client_config,
     )
     return SKLearn(
-        framework_version="0.23-1",
+        framework_version="1.2-1",
         entry_point=_ENTRY_POINT_PATH,
         dependencies=[sdk_tar],
         role=execution_role,

tests/integ/sagemaker/remote_function/conftest.py

@@ -80,7 +80,7 @@
     "channels:\n"
     "  - defaults\n"
     "dependencies:\n"
-    "  - scipy=1.7.3\n"
+    "  - scipy=1.10.1\n"
     "  - pip:\n"
     "    - /sagemaker-{sagemaker_version}.tar.gz\n"
     "prefix: /opt/conda/bin/conda\n"

tests/scripts/run-notebook-test.sh

@@ -58,9 +58,22 @@ for env in base /home/ec2-user/anaconda3/envs/*; do
 
     sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/activate "$env"'
 
+    if [ $env = base ]; then
+        ENV_PYTHON=python
+    else
+        ENV_PYTHON="$env/bin/python"
+    fi
+
+    PYTHON_VERSION=$($ENV_PYTHON -c"import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))")
+    if [ $PYTHON_VERSION -lt 38 ]; then
+        echo "Skipping because $env uses py$PYTHON_VERSION which is incompatible with SageMaker Python SDK"
+        sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/deactivate'
+        continue
+    fi
+
     echo "Updating SageMaker Python SDK..."
-    pip install --upgrade pip
-    pip install "$TARBALL_DIRECTORY/sagemaker.tar.gz"
+    $ENV_PYTHON -m pip install --upgrade pip
+    $ENV_PYTHON -m pip install "$TARBALL_DIRECTORY/sagemaker.tar.gz"
 
     sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/deactivate'
 

tests/unit/sagemaker/deserializers/test_deserializers.py

@@ -119,8 +119,9 @@ def test_numpy_deserializer_from_json(numpy_deserializer):
 # Sadly, ragged arrays work fine in JSON (giving us a 1D array of Python lists)
 def test_numpy_deserializer_from_json_ragged(numpy_deserializer):
     stream = io.BytesIO(b"[[1,2,3],\n[4,5,6,7]]")
-    array = numpy_deserializer.deserialize(stream, "application/json")
-    assert np.array_equal(array, np.array([[1, 2, 3], [4, 5, 6, 7]]))
+    with pytest.raises(ValueError) as error:
+        numpy_deserializer.deserialize(stream, "application/json")
+    assert "requested array has an inhomogeneous shape" in str(error)
 
 
 def test_numpy_deserializer_from_json_alpha():