No public description

Signed-off-by: Googler <[email protected]>
PiperOrigin-RevId: 821935037

Author: Googler

components/google-cloud/RELEASE.md

@@ -1,5 +1,7 @@
 ## Upcoming release
 
+* Add `autotuning_config` and `cohort` parameters to Dataproc v1 batch components.
+
 ## Release 2.21.0
 
 * Add psc_interface_config to v1 GCPC custom job components/utils

components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_pyspark_batch/component.py

@@ -32,6 +32,8 @@ def dataproc_create_pyspark_batch(
     container_image: str = '',
     runtime_config_version: str = '',
     runtime_config_properties: Dict[str, str] = {},
+    autotuning_config: Dict[str, str] = {},
+    cohort: str = '',
     service_account: str = '',
     network_tags: List[str] = [],
     kms_key: str = '',
@@ -56,6 +58,8 @@ def dataproc_create_pyspark_batch(
       container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used.
       runtime_config_version: Version of the batch runtime.
       runtime_config_properties: Runtime configuration for the workload.
+      autotuning_config: Autotuning configuration for the workload.
+      cohort: Cohort identifier for the workload.
       service_account: Service account that is used to execute the workload.
       network_tags: Tags used for network traffic control.
       kms_key: The Cloud KMS key to use for encryption.
@@ -100,6 +104,11 @@ def dataproc_create_pyspark_batch(
               '"',
               ', "properties": ',
               runtime_config_properties,
+              ', "autotuning_config": ',
+              autotuning_config,
+              ', "cohort": "',
+              cohort,
+              '"',
               '}',
               ', "environment_config": {',
               '"execution_config": {',

components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_batch/component.py

@@ -31,6 +31,8 @@ def dataproc_create_spark_batch(
     container_image: str = '',
     runtime_config_version: str = '',
     runtime_config_properties: Dict[str, str] = {},
+    autotuning_config: Dict[str, str] = {},
+    cohort: str = '',
     service_account: str = '',
     network_tags: List[str] = [],
     kms_key: str = '',
@@ -56,6 +58,8 @@ def dataproc_create_spark_batch(
       container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used.
       runtime_config_version: Version of the batch runtime.
       runtime_config_properties: Runtime configuration for the workload.
+      autotuning_config: Autotuning configuration for the workload.
+      cohort: Cohort identifier for the workload.
       service_account: Service account that is used to execute the workload.
       network_tags: Tags used for network traffic control.
       kms_key: The Cloud KMS key to use for encryption.
@@ -100,6 +104,11 @@ def dataproc_create_spark_batch(
               '"',
               ', "properties": ',
               runtime_config_properties,
+              ', "autotuning_config": ',
+              autotuning_config,
+              ', "cohort": "',
+              cohort,
+              '"',
               '}',
               ', "environment_config": {',
               '"execution_config": {',

components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_r_batch/component.py

@@ -31,6 +31,8 @@ def dataproc_create_spark_r_batch(
     container_image: str = '',
     runtime_config_version: str = '',
     runtime_config_properties: Dict[str, str] = {},
+    autotuning_config: Dict[str, str] = {},
+    cohort: str = '',
     service_account: str = '',
     network_tags: List[str] = [],
     kms_key: str = '',
@@ -54,6 +56,8 @@ def dataproc_create_spark_r_batch(
       container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used.
       runtime_config_version: Version of the batch runtime.
       runtime_config_properties: Runtime configuration for the workload.
+      autotuning_config: Autotuning configuration for the workload.
+      cohort: Cohort identifier for the workload.
       service_account: Service account that is used to execute the workload.
       network_tags: Tags used for network traffic control.
       kms_key: The Cloud KMS key to use for encryption.
@@ -96,6 +100,11 @@ def dataproc_create_spark_r_batch(
               '"',
               ', "properties": ',
               runtime_config_properties,
+              ', "autotuning_config": ',
+              autotuning_config,
+              ', "cohort": "',
+              cohort,
+              '"',
               '}',
               ', "environment_config": {',
               '"execution_config": {',

components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_sql_batch/component.py

@@ -30,6 +30,8 @@ def dataproc_create_spark_sql_batch(
     container_image: str = '',
     runtime_config_version: str = '',
     runtime_config_properties: Dict[str, str] = {},
+    autotuning_config: Dict[str, str] = {},
+    cohort: str = '',
     service_account: str = '',
     network_tags: List[str] = [],
     kms_key: str = '',
@@ -52,6 +54,8 @@ def dataproc_create_spark_sql_batch(
       container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used.
       runtime_config_version: Version of the batch runtime.
       runtime_config_properties: Runtime configuration for the workload.
+      autotuning_config: Autotuning configuration for the workload.
+      cohort: Cohort identifier for the workload.
       service_account: Service account that is used to execute the workload.
       network_tags: Tags used for network traffic control.
       kms_key: The Cloud KMS key to use for encryption.
@@ -93,6 +97,11 @@ def dataproc_create_spark_sql_batch(
               '"',
               ', "properties": ',
               runtime_config_properties,
+              ', "autotuning_config": ',
+              autotuning_config,
+              ', "cohort": "',
+              cohort,
+              '"',
               '}',
               ', "environment_config": {',
               '"execution_config": {',