Improve EMR tutorials #193

Author: igorborgest

awswrangler/emr.py

@@ -649,10 +649,29 @@ def create_cluster(  # pylint: disable=too-many-arguments,too-many-locals,unused
     --------
     Minimal Example
 
+    >>> import awswrangler as wr
     >>> cluster_id = wr.emr.create_cluster("SUBNET_ID")
 
-    Minimal Exmaple on Docker
+    Minimal Example With Custom Classification
 
+    >>> import awswrangler as wr
+    >>> cluster_id = wr.emr.create_cluster(
+    >>> subnet_id="SUBNET_ID",
+    >>> custom_classifications=[
+    >>>         {
+    >>>             "Classification": "livy-conf",
+    >>>             "Properties": {
+    >>>                 "livy.spark.master": "yarn",
+    >>>                 "livy.spark.deploy-mode": "cluster",
+    >>>                 "livy.server.session.timeout": "16h",
+    >>>             },
+    >>>         }
+    >>>     ],
+    >>> )
+
+    Minimal Example on Docker
+
+    >>> import awswrangler as wr
     >>> cluster_id = wr.emr.create_cluster(
     >>>     subnet_id="SUBNET_ID",
     >>>     spark_docker=True,

testing/test_awswrangler/test_emr.py

@@ -161,7 +161,7 @@ def test_docker(cloudformation_outputs):
         subnet_id=cloudformation_outputs["SubnetId"],
         docker=True,
         spark_docker=True,
-        spark_docker_image="787535711150.dkr.ecr.us-east-1.amazonaws.com/docker-emr:docker-emr",
+        spark_docker_image="123456789123.dkr.ecr.us-east-1.amazonaws.com/docker-emr:docker-emr",
         hive_docker=True,
         ecr_credentials_step=True,
         custom_classifications=[
@@ -174,8 +174,8 @@ def test_docker(cloudformation_outputs):
                 },
             }
         ],
-        steps=[wr.emr.build_step("spark-submit --deploy-mode cluster s3://igor-tavares/emr.py")],
+        steps=[wr.emr.build_step("spark-submit --deploy-mode cluster s3://bucket/emr.py")],
     )
-    wr.emr.submit_step(cluster_id=cluster_id, command="spark-submit --deploy-mode cluster s3://igor-tavares/emr.py")
+    wr.emr.submit_step(cluster_id=cluster_id, command="spark-submit --deploy-mode cluster s3://bucket/emr.py")
     wr.emr.update_ecr_credentials(cluster_id=cluster_id)
     wr.emr.terminate_cluster(cluster_id=cluster_id)

tutorials/16 - EMR & Docker.ipynb

@@ -201,7 +201,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "step_id = wr.emr.submit_step(cluster_id, command=f\"spark-submit --deploy-mode cluster s3://{bucket}/test_docker.py\")"
+    "step_id = wr.emr.submit_step(\n",
+    "    cluster_id=cluster_id,\n",
+    "    command=f\"spark-submit --deploy-mode cluster s3://{bucket}/test_docker.py\"\n",
+    ")"
    ]
   },
   {
@@ -266,4 +269,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}