Update spark runtime version (#3638)

Author: ricardrao

sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb

@@ -24,8 +24,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from azure.ai.ml import MLClient, spark, Input, Output\n",
+    "from azure.ai.ml import MLClient, spark\n",
     "from azure.identity import DefaultAzureCredential\n",
+    "import time\n",
     "\n",
     "subscription_id = \"<SUBSCRIPTION_ID>\"\n",
     "resource_group = \"<RESOURCE_GROUP>\"\n",
@@ -49,30 +50,42 @@
     "    },\n",
     ")\n",
     "\n",
+    "# Create or update the Spark job\n",
     "returned_spark_job = ml_client.jobs.create_or_update(spark_job)\n",
     "\n",
     "print(returned_spark_job.id)\n",
     "# Wait until the job completes\n",
-    "ml_client.jobs.stream(returned_spark_job.name)"
+    "job_name = returned_spark_job.name\n",
+    "while True:\n",
+    "    job = ml_client.jobs.get(job_name)\n",
+    "    print(f\"Current job status: {job.status}\")\n",
+    "    if job.status in [\"Completed\", \"Failed\", \"Canceled\"]:\n",
+    "        break\n",
+    "    time.sleep(10)\n",
+    "\n",
+    "print(f\"Final job status: {job.status}\")"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.10 - SDK V2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python310-sdkv2"
+   "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
-   "version": "3.7.10"
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.10"
   },
-  "orig_nbformat": 4,
-  "vscode": {
-   "interpreter": {
-    "hash": "6aeff17a1aa7735c2f7cb3a6d691fe1b4d4c3b8d2d650f644ad0f24e1b8e3f3f"
-   }
-  }
+  "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 2

sdk/python/jobs/spark/setup_spark.sh

@@ -56,7 +56,10 @@ then
 	GEN2_STORAGE_ACCOUNT_NAME=${RESOURCE_GROUP}gen2vnet
 	ADLS_CONTAINER_NAME="gen2containervnet"
 
+	EXIST=$(az storage account check-name --name $DEFAULT_STORAGE_ACCOUNT --query nameAvailable)
+	if [ "$EXIST" = "true" ]; then
 	az storage account create -n $DEFAULT_STORAGE_ACCOUNT -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS
+	fi
 
 	az storage account create -n $AZURE_STORAGE_ACCOUNT -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS
 	az storage container create -n $BLOB_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT