Renyix/data import sdkv2 (Azure#29143)

* Add azureml data import verb sdk support

* Create data import pipeline job

* Update doc string

* data import - add unit tests

* data import - skip serverless compute validation

* data import - fix unit tests

* data import - fix DataImportSchema import build error

* data import - remove used JobOperations to fix pylint build check

* add __init__ to fix build checks

* data import - fix line too long build error

* data import - generate recording files

* Fix Run Black build check error

* Refine data import job submission

* Add back some data import parameters

* Mark data import classes as experimental

* Resolve code review comments

* Resolve code review comments

* Remove unused import to fix pyint

* data import - update e2e test recording file

* Add back @experimental for DataImport class and disable asset_name filter

* Update e2e test recording file

* Fix unused 'name' pyint build check

* Merge load_data_import into load_data

* Revert unexpected change

* Remove load_data_import func and update load_data doc string

* Update data import e2e test recording file

* Enable back asset_name filter as mfe 2023-02-01-preview is in place

Author: xiongrenyi

sdk/ml/azure-ai-ml/azure/ai/ml/__init__.py

@@ -22,7 +22,6 @@
     load_component,
     load_compute,
     load_data,
-    load_data_import,
     load_datastore,
     load_environment,
     load_job,
@@ -55,7 +54,6 @@
     "load_component",
     "load_compute",
     "load_data",
-    "load_data_import",
     "load_datastore",
     "load_model",
     "load_environment",

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_data_import/data_import.py

@@ -67,16 +67,18 @@ def _load(
         yaml_path: Optional[Union[PathLike, str]] = None,
         params_override: Optional[list] = None,
         **kwargs,
-    ) -> "DataImport":
+    ) -> Union["Data", "DataImport"]:
         data = data or {}
         params_override = params_override or []
         context = {
             BASE_PATH_CONTEXT_KEY: Path(yaml_path).parent if yaml_path else Path("./"),
             PARAMS_OVERRIDE_KEY: params_override,
         }
-        data_import = DataImport._load_from_dict(yaml_data=data, context=context, **kwargs)
 
-        return data_import
+        if "source" in data:
+            return DataImport._load_from_dict(yaml_data=data, context=context, **kwargs)
+
+        return Data._load_from_dict(yaml_data=data, context=context, **kwargs)
 
     @classmethod
     def _load_from_dict(cls, yaml_data: Dict, context: Dict, **kwargs) -> "DataImport":

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_load_functions.py

@@ -427,7 +427,7 @@ def load_data(
     *,
     relative_origin: Optional[str] = None,
     **kwargs,
-) -> Data:
+) -> Union[Data, DataImport]:
     """Construct a data object from yaml file.
 
     :param source: The local yaml source of a data object. Must be either a
@@ -447,39 +447,8 @@ def load_data(
     :type params_override: List[Dict]
     :raises ~azure.ai.ml.exceptions.ValidationException: Raised if Data cannot be successfully validated.
         Details will be provided in the error message.
-    :return: Constructed data object.
-    :rtype: Data
-    """
-    return load_common(Data, source, relative_origin, **kwargs)
-
-
-def load_data_import(
-    source: Union[str, PathLike, IO[AnyStr]],
-    *,
-    relative_origin: Optional[str] = None,
-    **kwargs,
-) -> DataImport:
-    """Construct a data object from yaml file.
-
-    :param source: The local yaml source of a data object. Must be either a
-        path to a local file, or an already-open file.
-        If the source is a path, it will be open and read.
-        An exception is raised if the file does not exist.
-        If the source is an open file, the file will be read directly,
-        and an exception is raised if the file is not readable.
-    :type source: Union[PathLike, str, io.TextIOWrapper]
-    :param relative_origin: The origin to be used when deducing
-        the relative locations of files referenced in the parsed yaml.
-        Defaults to the inputted source's directory if it is a file or file path input.
-        Defaults to "./" if the source is a stream input with no name value.
-    :type relative_origin: str
-    :param params_override: Fields to overwrite on top of the yaml file.
-        Format is [{"field1": "value1"}, {"field2": "value2"}]
-    :type params_override: List[Dict]
-    :raises ~azure.ai.ml.exceptions.ValidationException: Raised if DataImport cannot be successfully validated.
-        Details will be provided in the error message.
-    :return: Constructed data_import object.
-    :rtype: DataImport
+    :return: Constructed Data or DataImport object.
+    :rtype: Union[Data, DataImport]
     """
     return load_common(DataImport, source, relative_origin, **kwargs)
 

sdk/ml/azure-ai-ml/azure/ai/ml/operations/_data_operations.py

@@ -409,7 +409,7 @@ def show_materialization_status(
         """
 
         # TODO: Add back 'asset_name=name' filter once client switches to mfe 2023-02-01-preview and above
-        return self._job_operation.list(job_type="Pipeline", tag=name, list_view_type=list_view_type)
+        return self._job_operation.list(job_type="Pipeline", asset_name=name, list_view_type=list_view_type)
 
     # @monitor_with_activity(logger, "Data.Validate", ActivityType.INTERNALCALL)
     def _validate(self, data: Data) -> Union[List[str], None]:

sdk/ml/azure-ai-ml/tests/data_import/e2etests/test_data_import.py

@@ -1,7 +1,7 @@
 import pytest
 from devtools_testutils import AzureRecordedTestCase
 
-from azure.ai.ml import MLClient, load_data_import
+from azure.ai.ml import MLClient, load_data
 from azure.ai.ml.entities import PipelineJob, DataImport
 from azure.ai.ml.entities._inputs_outputs.external_data import Database
 
@@ -14,7 +14,7 @@ class TestDataImport(AzureRecordedTestCase):
     # Please set ML_TENANT_ID in your environment variables when recording this test.
     # It will to help sanitize RequestBody.Studio.endpoint for job creation request.
     def test_data_import(self, client: MLClient) -> None:
-        data_import = load_data_import(
+        data_import = load_data(
             source="./tests/test_configs/data_import/data_import_e2e.yaml",
         )
         pipeline_job: PipelineJob = client.data.import_data(data_import)

sdk/ml/azure-ai-ml/tests/data_import/unittests/test_data_import.py

@@ -1,18 +1,17 @@
 import pytest
 
-from azure.ai.ml import load_data_import
+from azure.ai.ml import load_data
 from azure.ai.ml.data_transfer import import_data
 from azure.ai.ml.entities import DataImport
 from azure.ai.ml.entities._inputs_outputs import Output
 from azure.ai.ml.entities._inputs_outputs.external_data import Database, FileSystem
-from azure.ai.ml.entities._job.data_transfer.data_transfer_job import DataTransferImportJob
 
 
 @pytest.mark.unittest
 @pytest.mark.data_import_test
 class TestDataImport:
     def test_data_import_database(self):
-        data_import1 = load_data_import(source="./tests/test_configs/data_import/data_import_database.yaml")
+        data_import1 = load_data(source="./tests/test_configs/data_import/data_import_database.yaml")
         data_import2 = DataImport(
             name="my_azuresqldb_asset",
             type="mltable",
@@ -33,7 +32,7 @@ def test_data_import_database(self):
         assert data_import1.source.connection == data_import2.source.connection
 
     def test_data_import_file_system(self):
-        data_import = load_data_import(source="./tests/test_configs/data_import/data_import_file_system.yaml")
+        data_import = load_data(source="./tests/test_configs/data_import/data_import_file_system.yaml")
         import_job = import_data(
             source=FileSystem(path="test1/*", connection="azureml:my_s3_connection"),
             outputs={

sdk/ml/azure-ai-ml/tests/recordings/data_import/e2etests/test_data_import.pyTestDataImporttest_data_import.json

@@ -1,7 +1,7 @@
 {
   "Entries": [
     {
-      "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/jobs/000000000000000000000?api-version=2022-12-01-preview",
+      "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/jobs/000000000000000000000?api-version=2023-02-01-preview",
       "RequestMethod": "PUT",
       "RequestHeaders": {
         "Accept": "application/json",
@@ -55,22 +55,22 @@
       "StatusCode": 201,
       "ResponseHeaders": {
         "Cache-Control": "no-cache",
-        "Content-Length": "2431",
+        "Content-Length": "2458",
         "Content-Type": "application/json; charset=utf-8",
-        "Date": "Fri, 03 Mar 2023 00:02:58 GMT",
+        "Date": "Sat, 04 Mar 2023 17:16:02 GMT",
         "Expires": "-1",
-        "Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/jobs/000000000000000000000?api-version=2022-12-01-preview",
+        "Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/jobs/000000000000000000000?api-version=2023-02-01-preview",
         "Pragma": "no-cache",
         "request-context": "appId=cid-v1:512cc15a-13b5-415b-bfd0-dce7accb6bb1",
-        "Server-Timing": "traceparent;desc=\u002200-28e082e30a9316b00c3313740706bec6-7d8ef3f3b1d0e411-01\u0022",
+        "Server-Timing": "traceparent;desc=\u002200-77a768c16dccf36b76f68d283a64790c-17f112882b45e4d7-01\u0022",
         "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
-        "x-aml-cluster": "vienna-test-westus2-02",
+        "x-aml-cluster": "vienna-test-westus2-01",
         "X-Content-Type-Options": "nosniff",
-        "x-ms-correlation-request-id": "34732156-6bf6-4c79-bc18-5e6aca254a7f",
+        "x-ms-correlation-request-id": "cb419d93-7c09-45e8-b365-1d21cc0f8b3d",
         "x-ms-ratelimit-remaining-subscription-writes": "1199",
         "x-ms-response-type": "standard",
-        "x-ms-routing-request-id": "WESTUS:20230303T000259Z:34732156-6bf6-4c79-bc18-5e6aca254a7f",
-        "x-request-time": "4.607"
+        "x-ms-routing-request-id": "WESTCENTRALUS:20230304T171603Z:cb419d93-7c09-45e8-b365-1d21cc0f8b3d",
+        "x-request-time": "4.352"
       },
       "ResponseBody": {
         "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/jobs/000000000000000000000",
@@ -119,6 +119,7 @@
           "isArchived": false,
           "identity": null,
           "componentId": null,
+          "notificationSetting": null,
           "jobType": "Pipeline",
           "settings": {
             "force_rerun": true,
@@ -151,7 +152,7 @@
           "sourceJobId": null
         },
         "systemData": {
-          "createdAt": "2023-03-03T00:02:58.7419134\u002B00:00",
+          "createdAt": "2023-03-04T17:16:02.9271689\u002B00:00",
           "createdBy": "Firstname Lastname",
           "createdByType": "User"
         }