Spark examples to spark 3.4

vaibhj · vaibhj · commit 48a013aaae3e · 2024-12-13T10:20:49.000-08:00
diff --git a/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml
@@ -22,4 +22,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml
@@ -25,4 +25,4 @@ jobs:
     
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml
@@ -25,4 +25,4 @@ jobs:
     
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
diff --git a/cli/jobs/spark/serverless-spark-standalone-default-identity.yml b/cli/jobs/spark/serverless-spark-standalone-default-identity.yml
@@ -31,5 +31,5 @@ args: >-
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.3"
+  runtime_version: "3.4"
   
diff --git a/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml b/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.3"
+  runtime_version: "3.4"
   
diff --git a/cli/jobs/spark/serverless-spark-standalone-user-identity.yml b/cli/jobs/spark/serverless-spark-standalone-user-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.3"
+  runtime_version: "3.4"
   
diff --git a/cli/monitoring/advanced-model-monitoring.yaml b/cli/monitoring/advanced-model-monitoring.yaml
@@ -17,7 +17,7 @@ create_monitor:
 
   compute: 
     instance_type: standard_e4s_v3
-    runtime_version: "3.3"
+    runtime_version: "3.4"
 
   monitoring_target:
     ml_task: classification
diff --git a/cli/monitoring/custom-monitoring.yaml b/cli/monitoring/custom-monitoring.yaml
@@ -10,7 +10,7 @@ trigger:
 create_monitor:
   compute:
     instance_type: "standard_e8s_v3"
-    runtime_version: "3.3"
+    runtime_version: "3.4"
 
   monitoring_signals:
     customSignal:
diff --git a/cli/monitoring/generation-safety-quality-monitoring.yaml b/cli/monitoring/generation-safety-quality-monitoring.yaml
@@ -22,7 +22,7 @@ trigger:
 create_monitor:
   compute:
     instance_type: "standard_e8s_v3"
-    runtime_version: "3.3"
+    runtime_version: "3.4"
   monitoring_target:
     ml_task: classification
     endpoint_deployment_id: fake_endpoint_id
diff --git a/cli/monitoring/model-monitoring-with-collected-data.yaml b/cli/monitoring/model-monitoring-with-collected-data.yaml
@@ -16,7 +16,7 @@ trigger:
 create_monitor:
   compute: 
     instance_type: standard_e4s_v3
-    runtime_version: "3.3"
+    runtime_version: "3.4"
   monitoring_target:
     ml_task: classification
     endpoint_deployment_id: azureml:fraud-detection-endpoint:fraud-detection-deployment
diff --git a/cli/monitoring/out-of-box-monitoring.yaml b/cli/monitoring/out-of-box-monitoring.yaml
@@ -17,7 +17,7 @@ create_monitor:
 
   compute: # specify a spark compute for monitoring job
     instance_type: standard_e4s_v3
-    runtime_version: "3.3"
+    runtime_version: "3.4"
 
   monitoring_target: 
     ml_task: classification # model task type: [classification, regression, question_answering]
diff --git a/sdk/python/jobs/spark/setup_spark.sh b/sdk/python/jobs/spark/setup_spark.sh
@@ -156,7 +156,7 @@ else
 	az storage fs create -n $GEN2_FILE_SYSTEM --account-name $GEN2_STORAGE_NAME
 	az synapse workspace create --name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --storage-account $GEN2_STORAGE_NAME --file-system $GEN2_FILE_SYSTEM --sql-admin-login-user $SQL_ADMIN_LOGIN_USER --sql-admin-login-password $RANDOM_STRING --location $LOCATION
 	az role assignment create --role "Storage Blob Data Owner" --assignee $AML_USER_MANAGED_ID_OID --scope /subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$GEN2_STORAGE_NAME/blobServices/default/containers/$GEN2_FILE_SYSTEM
-	az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.3 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true
+	az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.4 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true
 	az synapse workspace firewall-rule create --name allowAll --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --start-ip-address 0.0.0.0 --end-ip-address 255.255.255.255
 	#</create_attached_resources>
 
diff --git a/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb
@@ -342,7 +342,7 @@
     "    spark_step.identity = ManagedIdentityConfiguration()\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "\n",
     "\n",
@@ -422,7 +422,7 @@
     "    spark_step.identity = UserIdentityConfiguration()\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "\n",
     "\n",
@@ -501,7 +501,7 @@
     "    spark_step.outputs.wrangled_data.mode = InputOutputModes.DIRECT\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "\n",
     "\n",
diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb
@@ -267,7 +267,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -329,7 +329,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -391,7 +391,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb
@@ -535,7 +535,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -769,7 +769,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
