auto ml job serialization failure fix (#37043)

* fixing sweep job service response parsing issue

* fixing sweep job service response parsing issue

* raising a work around for cmd job and spark job name lowering case

* fixing black issue

* fixing serialization issue in sweep job

* include black fix

* found and fix the issue in compute validation

* fixing a corrupt wokspace arm template

* fixing the failed test case

* updating the date for 1.18.0 release

* removing empty section

* adding in progress version details

* adding IDC code reviewers

* correcting the directory path

* lint fix

* adding change log entry

* addressing review comment

* fixing ut

* addressing api review comment

* adding a fix for subnet validation in compute instance

* my py fix

* fixing autoML serialization issue

* formatting

* moving auto ML job create API to 2024

Author: achauhan-scc

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/automl_job.py

@@ -8,7 +8,7 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, Optional, Union
 
-from azure.ai.ml._restclient.v2023_04_01_preview.models import (
+from azure.ai.ml._restclient.v2024_01_01_preview.models import (
     JobBase,
     MLTableJobInput,
     QueueSettings,

sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py

@@ -725,7 +725,8 @@ def _create_or_update_with_different_version_api(  # pylint: disable=name-too-lo
             service_client_operation = self.service_client_01_2024_preview.jobs
         if rest_job_resource.properties.job_type == RestJobType.PIPELINE:
             service_client_operation = self.service_client_01_2024_preview.jobs
-
+        if rest_job_resource.properties.job_type == RestJobType.AUTO_ML:
+            service_client_operation = self.service_client_01_2024_preview.jobs
         if rest_job_resource.properties.job_type == RestJobType.SWEEP:
             service_client_operation = self.service_client_01_2024_preview.jobs
 

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification.py

@@ -5,14 +5,14 @@
 import pytest
 
 from azure.ai.ml import UserIdentityConfiguration
-from azure.ai.ml._restclient.v2023_04_01_preview.models import (
+from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
+from azure.ai.ml._restclient.v2024_01_01_preview.models import (
     ClassificationPrimaryMetrics,
     LearningRateScheduler,
     MLTableJobInput,
     SamplingAlgorithmType,
     StochasticOptimizer,
 )
-from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
 from azure.ai.ml.automl import image_classification
 from azure.ai.ml.constants._common import AssetTypes
 from azure.ai.ml.entities._inputs_outputs import Input

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification_multilabel.py

@@ -5,14 +5,14 @@
 import pytest
 
 from azure.ai.ml import UserIdentityConfiguration
-from azure.ai.ml._restclient.v2023_04_01_preview.models import (
+from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
+from azure.ai.ml._restclient.v2024_01_01_preview.models import (
     ClassificationMultilabelPrimaryMetrics,
     LearningRateScheduler,
     MLTableJobInput,
     SamplingAlgorithmType,
     StochasticOptimizer,
 )
-from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
 from azure.ai.ml.automl import image_classification_multilabel
 from azure.ai.ml.constants._common import AssetTypes
 from azure.ai.ml.entities._inputs_outputs import Input

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py

@@ -5,18 +5,18 @@
 import pytest
 
 from azure.ai.ml import UserIdentityConfiguration
-from azure.ai.ml._restclient.v2023_04_01_preview.models import (
+from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
+from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType
+from azure.ai.ml._restclient.v2024_01_01_preview.models import (
     InstanceSegmentationPrimaryMetrics,
     LearningRateScheduler,
+    LogTrainingMetrics,
+    LogValidationLoss,
     MLTableJobInput,
     ModelSize,
     SamplingAlgorithmType,
     StochasticOptimizer,
-    LogTrainingMetrics,
-    LogValidationLoss,
 )
-from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
-from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType
 from azure.ai.ml.automl import image_instance_segmentation
 from azure.ai.ml.constants._common import AssetTypes
 from azure.ai.ml.entities._inputs_outputs import Input

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py

@@ -5,18 +5,18 @@
 import pytest
 
 from azure.ai.ml import UserIdentityConfiguration
-from azure.ai.ml._restclient.v2023_04_01_preview.models import (
+from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
+from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType
+from azure.ai.ml._restclient.v2024_01_01_preview.models import (
     LearningRateScheduler,
+    LogTrainingMetrics,
+    LogValidationLoss,
     MLTableJobInput,
     ModelSize,
     ObjectDetectionPrimaryMetrics,
     SamplingAlgorithmType,
     StochasticOptimizer,
-    LogTrainingMetrics,
-    LogValidationLoss,
 )
-from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
-from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType
 from azure.ai.ml.automl import image_object_detection
 from azure.ai.ml.constants._common import AssetTypes
 from azure.ai.ml.entities._inputs_outputs import Input

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py

@@ -48,7 +48,7 @@
     ImageObjectDetection as RestImageObjectDetection,
 )
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ImageSweepSettings as RestImageSweepSettings
-from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import (
+from azure.ai.ml._restclient.v2024_01_01_preview.models._models_py3 import (
     InstanceSegmentationPrimaryMetrics,
     JobBase,
     LogVerbosity,

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_nlp_schema.py

@@ -8,12 +8,6 @@
 from azure.ai.ml import load_job
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import AutoMLJob as RestAutoMLJob
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import BanditPolicy as RestBanditPolicy
-from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import (
-    ClassificationPrimaryMetrics,
-    JobBase,
-    LogVerbosity,
-    MLTableJobInput,
-)
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import NlpFixedParameters as RestNlpFixedParameters
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import (
     NlpParameterSubspace as RestNlpParameterSubspace,
@@ -30,6 +24,12 @@
     TextClassificationMultilabel as RestTextClassificationMultilabel,
 )
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import TextNer as RestTextNer
+from azure.ai.ml._restclient.v2024_01_01_preview.models._models_py3 import (
+    ClassificationPrimaryMetrics,
+    JobBase,
+    LogVerbosity,
+    MLTableJobInput,
+)
 from azure.ai.ml._scope_dependent_operations import OperationScope
 from azure.ai.ml._utils.utils import dump_yaml_to_file, load_yaml, to_iso_duration_format_mins
 from azure.ai.ml.automl import (

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_tabular_schema.py

@@ -26,7 +26,6 @@
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import (
     ForecastingTrainingSettings as RestForecastingTrainingSettings,
 )
-from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import JobBase, LogVerbosity, MLTableJobInput
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import Regression as RestRegression
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import RegressionPrimaryMetrics
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import (
@@ -38,6 +37,7 @@
 from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import (
     TableVerticalLimitSettings as RestTableVerticalLimitSettings,
 )
+from azure.ai.ml._restclient.v2024_01_01_preview.models._models_py3 import JobBase, LogVerbosity, MLTableJobInput
 from azure.ai.ml._schema.automl.table_vertical.regression import AutoMLRegressionSchema
 from azure.ai.ml._scope_dependent_operations import OperationScope
 from azure.ai.ml._utils.utils import camel_to_snake, to_iso_duration_format_mins
@@ -47,11 +47,11 @@
 from azure.ai.ml.entities import Job
 from azure.ai.ml.entities._inputs_outputs import Input
 from azure.ai.ml.entities._job.automl.automl_job import AutoMLJob
-from azure.ai.ml.exceptions import ValidationException
 from azure.ai.ml.entities._job.automl.tabular.featurization_settings import TabularFeaturizationSettings
 from azure.ai.ml.entities._job.automl.tabular.forecasting_settings import ForecastingSettings
 from azure.ai.ml.entities._job.automl.tabular.limit_settings import TabularLimitSettings
 from azure.ai.ml.entities._job.automl.training_settings import TrainingSettings
+from azure.ai.ml.exceptions import ValidationException
 
 
 @pytest.fixture(autouse=True)

sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_classification_job.py

@@ -6,8 +6,9 @@
 import pytest
 
 from azure.ai.ml import UserIdentityConfiguration
-from azure.ai.ml._restclient.v2023_04_01_preview.models import CustomNCrossValidations, MLTableJobInput
+from azure.ai.ml._restclient.v2023_04_01_preview.models import CustomNCrossValidations
 from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
+from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput
 from azure.ai.ml.automl import ClassificationModels, ClassificationPrimaryMetrics, classification
 from azure.ai.ml.constants import TabularTrainingMode
 from azure.ai.ml.constants._common import AssetTypes