Spark upgrade (#3629)

kshitij-microsoft · web-flow · commit 65eceac7ff5c · 2025-06-27T14:45:49.000+05:30
diff --git a/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml b/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml
@@ -32,7 +32,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.3.0
+      runtime_version: 3.4.0
 
   count_word:
     type: spark
@@ -44,6 +44,6 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.3.0
+      runtime_version: 3.4.0
 
 
diff --git a/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml b/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml
@@ -36,7 +36,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.3.0
+      runtime_version: 3.4.0
 
     conf:
       spark.driver.cores: 2
@@ -71,7 +71,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.3.0
+      runtime_version: 3.4.0
 
     conf:
       spark.driver.cores: 2
diff --git a/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml
@@ -22,4 +22,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml
@@ -25,4 +25,4 @@ jobs:
     
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml
@@ -25,4 +25,4 @@ jobs:
     
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
diff --git a/cli/jobs/spark/serverless-spark-standalone-default-identity.yml b/cli/jobs/spark/serverless-spark-standalone-default-identity.yml
@@ -31,5 +31,5 @@ args: >-
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.3"
+  runtime_version: "3.4"
   
diff --git a/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml b/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.3"
+  runtime_version: "3.4"
   
diff --git a/cli/jobs/spark/serverless-spark-standalone-user-identity.yml b/cli/jobs/spark/serverless-spark-standalone-user-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.3"
+  runtime_version: "3.4"
   
diff --git a/cli/monitoring/advanced-model-monitoring.yaml b/cli/monitoring/advanced-model-monitoring.yaml
@@ -17,7 +17,7 @@ create_monitor:
 
   compute: 
     instance_type: standard_e4s_v3
-    runtime_version: "3.3"
+    runtime_version: "3.4"
 
   monitoring_target:
     ml_task: classification
diff --git a/cli/monitoring/azureml-e2e-model-monitoring/notebooks/model-monitoring-e2e.ipynb b/cli/monitoring/azureml-e2e-model-monitoring/notebooks/model-monitoring-e2e.ipynb
@@ -759,7 +759,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -794,7 +794,7 @@
     "\n",
     "# create the compute\n",
     "spark_compute = ServerlessSparkCompute(\n",
-    "    instance_type=\"standard_e4s_v3\", runtime_version=\"3.3\"\n",
+    "    instance_type=\"standard_e4s_v3\", runtime_version=\"3.4\"\n",
     ")\n",
     "\n",
     "# specify your online endpoint deployment\n",
@@ -838,7 +838,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -908,7 +908,7 @@
     "\n",
     "# create your compute\n",
     "spark_compute = ServerlessSparkCompute(\n",
-    "    instance_type=\"standard_e4s_v3\", runtime_version=\"3.3\"\n",
+    "    instance_type=\"standard_e4s_v3\", runtime_version=\"3.4\"\n",
     ")\n",
     "\n",
     "# specify the online deployment (if you have one)\n",
diff --git a/cli/monitoring/custom-monitoring.yaml b/cli/monitoring/custom-monitoring.yaml
@@ -10,7 +10,7 @@ trigger:
 create_monitor:
   compute:
     instance_type: "standard_e8s_v3"
-    runtime_version: "3.3"
+    runtime_version: "3.4"
 
   monitoring_signals:
     customSignal:
diff --git a/cli/monitoring/generation-safety-quality-monitoring.yaml b/cli/monitoring/generation-safety-quality-monitoring.yaml
@@ -22,7 +22,7 @@ trigger:
 create_monitor:
   compute:
     instance_type: "standard_e8s_v3"
-    runtime_version: "3.3"
+    runtime_version: "3.4"
   monitoring_target:
     ml_task: classification
     endpoint_deployment_id: fake_endpoint_id
diff --git a/cli/monitoring/model-monitoring-with-collected-data.yaml b/cli/monitoring/model-monitoring-with-collected-data.yaml
@@ -16,7 +16,7 @@ trigger:
 create_monitor:
   compute: 
     instance_type: standard_e4s_v3
-    runtime_version: "3.3"
+    runtime_version: "3.4"
   monitoring_target:
     ml_task: classification
     endpoint_deployment_id: azureml:fraud-detection-endpoint:fraud-detection-deployment
diff --git a/cli/monitoring/out-of-box-monitoring.yaml b/cli/monitoring/out-of-box-monitoring.yaml
@@ -17,7 +17,7 @@ create_monitor:
 
   compute: # specify a spark compute for monitoring job
     instance_type: standard_e4s_v3
-    runtime_version: "3.3"
+    runtime_version: "3.4"
 
   monitoring_target: 
     ml_task: classification # model task type: [classification, regression, question_answering]
diff --git a/sdk/python/featurestore_sample/automation-test/test_featurestore_cli_samples.ipynb b/sdk/python/featurestore_sample/automation-test/test_featurestore_cli_samples.ipynb
@@ -69,7 +69,7 @@
     "    executor_instances=1,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    conf={\"spark.synapse.library.python.env\": contents},\n",
     ")\n",
diff --git a/sdk/python/featurestore_sample/automation-test/test_featurestore_sdk_samples.ipynb b/sdk/python/featurestore_sample/automation-test/test_featurestore_sdk_samples.ipynb
@@ -69,7 +69,7 @@
     "    executor_instances=1,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    conf={\"spark.synapse.library.python.env\": contents},\n",
     ")\n",
diff --git a/sdk/python/featurestore_sample/automation-test/test_featurestore_vnet_samples.ipynb b/sdk/python/featurestore_sample/automation-test/test_featurestore_vnet_samples.ipynb
@@ -69,7 +69,7 @@
     "    executor_instances=1,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    conf={\"spark.synapse.library.python.env\": contents},\n",
     ")\n",
diff --git a/sdk/python/featurestore_sample/project/fraud_model/pipelines/batch_inference_pipeline.yaml b/sdk/python/featurestore_sample/project/fraud_model/pipelines/batch_inference_pipeline.yaml
@@ -28,7 +28,7 @@ jobs:
       observation_data_format: parquet
     resources:
       instance_type: standard_e4s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
     outputs:
       output_data:
     conf:
diff --git a/sdk/python/featurestore_sample/project/fraud_model/pipelines/training_pipeline.yaml b/sdk/python/featurestore_sample/project/fraud_model/pipelines/training_pipeline.yaml
@@ -28,7 +28,7 @@ jobs:
       observation_data_format: parquet
     resources:
       instance_type: standard_e4s_v3
-      runtime_version: "3.3"
+      runtime_version: "3.4"
     outputs:
       output_data:
     conf:
diff --git a/sdk/python/jobs/pipelines/1i_pipeline_with_spark_nodes/pipeline_with_spark_nodes.ipynb b/sdk/python/jobs/pipelines/1i_pipeline_with_spark_nodes/pipeline_with_spark_nodes.ipynb
@@ -171,7 +171,7 @@
     "    kmeans_clustering = spark_kmeans(file_input=train_data)\n",
     "    kmeans_clustering.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "    kmeans_clustering.outputs.output.mode = InputOutputModes.DIRECT\n",
     "\n",
diff --git a/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb b/sdk/python/jobs/pipelines/1k_demand_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb
@@ -601,7 +601,7 @@
     "|Property|Description|\n",
     "|:-|:-|\n",
     "| **instance_type**            | A key that defines the compute instance type to be used for the serverless Spark compute. The following instance types are currently supported:<ul><li>`Standard_E4S_V3`</li><li>`Standard_E8S_V3`</li><li>`Standard_E16S_V3`</li><li>`Standard_E32S_V3`</li><li>`Standard_E64S_V3`</li></ul>|\n",
-    "| **runtime_version**          | A key that defines the Spark runtime version. The following Spark runtime versions are currently supported:<ul><li>`3.3.0`</li><li>`3.4.0`</li></ul> |\n",
+    "| **runtime_version**          | A key that defines the Spark runtime version. The following Spark runtime versions are currently supported:<ul><li>`3.4.0`</li></ul> |\n",
     "| **driver_cores**       | The he number of cores allocated for the Spark driver. |\n",
     "| **driver_memory**      | The allocated memory for the Spark exedriver, with a size unit suffix `k`, `m`, `g` or `t` (for example, `512m`, `2g`). |\n",
     "| **executor_cores**     | The number of cores allocated for the Spark executor. |\n",
@@ -639,7 +639,7 @@
     "if USE_PARTITIONING_COMPONENT:\n",
     "    spark_parameters = dict(\n",
     "        instance_type=\"Standard_E4S_V3\",\n",
-    "        runtime_version=\"3.3.0\",\n",
+    "        runtime_version=\"3.4.0\",\n",
     "        driver_cores=1,\n",
     "        driver_memory=\"2g\",\n",
     "        executor_cores=2,\n",
@@ -704,7 +704,7 @@
     "\n",
     "        partition_step.resources = {\n",
     "            \"instance_type\": spark_parameters.get(\"instance_type\", \"Standard_E4S_V3\"),\n",
-    "            \"runtime_version\": str(spark_parameters.get(\"runtime_version\", \"3.3.0\")),\n",
+    "            \"runtime_version\": str(spark_parameters.get(\"runtime_version\", \"3.4.0\")),\n",
     "        }\n",
     "        partition_step.conf = {\n",
     "            \"spark.driver.cores\": spark_parameters.get(\"driver_cores\", 1),\n",
diff --git a/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb b/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb
@@ -45,7 +45,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     ")\n",
     "\n",
diff --git a/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb
@@ -342,7 +342,7 @@
     "    spark_step.identity = ManagedIdentityConfiguration()\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "\n",
     "\n",
@@ -422,7 +422,7 @@
     "    spark_step.identity = UserIdentityConfiguration()\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "\n",
     "\n",
@@ -501,7 +501,7 @@
     "    spark_step.outputs.wrangled_data.mode = InputOutputModes.DIRECT\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    }\n",
     "\n",
     "\n",
diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb
@@ -267,7 +267,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -329,7 +329,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -391,7 +391,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb
@@ -535,7 +535,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -769,7 +769,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.3.0\",\n",
+    "        \"runtime_version\": \"3.4.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
