change: bump boto3 and smdebug_rulesconfig versions for reinvent and enable data parallel integ tests

Author: Dan Choi

setup.py

@@ -34,12 +34,12 @@ def read_version():
 # Declare minimal set for installation
 required_packages = [
     "attrs",
-    "boto3>=1.16.27",
+    "boto3>=1.16.32",
     "google-pasta",
     "numpy>=1.9.0",
     "protobuf>=3.1",
     "protobuf3-to-dict>=0.1.5",
-    "smdebug_rulesconfig",
+    "smdebug_rulesconfig>=1.0.0",
     "importlib-metadata>=1.4.0",
     "packaging>=20.0",
 ]

src/sagemaker/image_uris.py

@@ -91,9 +91,13 @@ def retrieve(
     if _should_auto_select_container_version(instance_type, distribution):
         container_versions = {
             "tensorflow-2.3-gpu-py37": "cu110-ubuntu18.04-v3",
+            "tensorflow-2.3.1-gpu-py37": "cu110-ubuntu18.04",
             "tensorflow-1.15-gpu-py37": "cu110-ubuntu18.04-v8",
+            "tensorflow-1.15.4-gpu-py37": "cu110-ubuntu18.04",
             "mxnet-1.8-gpu-py37": "cu110-ubuntu16.04-v1",
+            "mxnet-1.8.0-gpu-py37": "cu110-ubuntu16.04",
             "pytorch-1.6-gpu-py36": "cu110-ubuntu18.04-v3",
+            "pytorch-1.6.0-gpu-py36": "cu110-ubuntu18.04",
         }
         key = "-".join([framework, tag])
         if key in container_versions:

tests/integ/test_smdataparallel_pt.py

@@ -14,7 +14,6 @@
 
 import os
 
-import pytest
 import sagemaker.utils
 import tests.integ as integ
 
@@ -27,19 +26,21 @@
 )
 
 
-@pytest.mark.skip(
-    reason="SMDistributedDataParallel-enabled DLC isn't publicly released hence not accessible for this test"
-)
-def test_smdataparallel_pt_mnist(sagemaker_session):
+def test_smdataparallel_pt_mnist(
+    sagemaker_session,
+    pytorch_training_latest_version,
+    pytorch_training_latest_py_version,
+):
     job_name = sagemaker.utils.unique_name_from_base("pt-sm-distributed-dataparallel")
     estimator = PyTorch(
         entry_point="mnist_pt.py",
         role="SageMakerRole",
-        image_uri="redacted",
         source_dir=smdataparallel_dir,
         instance_count=2,
         instance_type="ml.p3.16xlarge",
         sagemaker_session=sagemaker_session,
+        framework_version=pytorch_training_latest_version,
+        py_version=pytorch_training_latest_py_version,
         distribution={"smdistributed": {"dataparallel": {"enabled": True}}},
     )
 

tests/integ/test_smdataparallel_tf.py

@@ -14,7 +14,6 @@
 
 import os
 
-import pytest
 import sagemaker.utils
 import tests.integ as integ
 
@@ -26,19 +25,21 @@
 )
 
 
-@pytest.mark.skip(
-    reason="SMDistributedDataParallel-enabled DLC isn't publicly released hence not accessible for this test"
-)
-def test_smdataparallel_tf_mnist(sagemaker_session):
+def test_smdataparallel_tf_mnist(
+    sagemaker_session,
+    tensorflow_training_latest_version,
+    tensorflow_training_latest_py_version,
+):
     job_name = sagemaker.utils.unique_name_from_base("tf-sm-distributed-dataparallel")
     estimator = TensorFlow(
         entry_point="mnist_tf.py",
         role="SageMakerRole",
-        image_uri="redacted",
         source_dir=smdataparallel_dir,
         instance_count=2,
         instance_type="ml.p3.16xlarge",
         sagemaker_session=sagemaker_session,
+        framework_version=tensorflow_training_latest_version,
+        py_version=tensorflow_training_latest_py_version,
         distribution={"smdistributed": {"dataparallel": {"enabled": True}}},
     )