[ml] support pe fqdns and st address prefixes (#36960)

* addressPrefixes should not be readonly. ARM swagger 30248

* update swagger restclient version of workspace related operations

* add new properties support for outbound rules in SDK

* managed_network has been GA for long time

* update changelog

* cspell on appGw and sphinx docs error

* linting pt2

---------

Co-authored-by: Josh Harrington <[email protected]>

Author: joshharrin

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -3,7 +3,10 @@
 ## 1.20.0 (unreleased)
 
 ### Features Added
-- When a workspace is created with `managed_network` enabled or has `public_network_access` set to disabled, the resources created with the workspace (Key Vault, Storage Account) will be set to have restricted network access settings. This is only applicable when the user does not specify existing resources.
+  - When a workspace is created with `managed_network` enabled or has `public_network_access` set to disabled, the resources created with the workspace (Key Vault, Storage Account) will be set to have restricted network access settings. This is only applicable when the user does not specify existing resources.
+  - Added support of `fqdns` property for managed network `PrivateEndpointDestination` outbound rule objects. Enabling the support of Application Gateway as a Private Endpoint target in the workspace managed network.
+  - Added support of `address_prefixes` property for managed network `ServiceTagDestination` outbound rule objects.
+  - Removed experimental tag from `managed_network` which is a GA feature.
 
 ## 1.19.0 (2024-07-29)
 

sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py

@@ -474,7 +474,7 @@ def __init__(
 
         self._workspaces = WorkspaceOperations(
             self._ws_operation_scope if registry_reference else self._operation_scope,
-            self._service_client_08_2023_preview,
+            self._service_client_07_2024_preview,
             self._operation_container,
             self._credential,
             requests_pipeline=self._requests_pipeline,
@@ -485,7 +485,7 @@ def __init__(
 
         self._workspace_outbound_rules = WorkspaceOutboundRuleOperations(
             self._operation_scope,
-            self._service_client_08_2023_preview,
+            self._service_client_07_2024_preview,
             self._operation_container,
             self._credential,
             **kwargs,
@@ -702,7 +702,7 @@ def __init__(
 
         self._featurestores = FeatureStoreOperations(
             self._operation_scope,
-            self._service_client_08_2023_preview,
+            self._service_client_07_2024_preview,
             self._operation_container,
             self._credential,
             **app_insights_handler_kwargs,  # type: ignore[arg-type]

sdk/ml/azure-ai-ml/azure/ai/ml/_restclient/v2024_07_01_preview/models/_models.py

@@ -26588,8 +26588,6 @@ def __init__(
 class ServiceTagDestination(msrest.serialization.Model):
     """Service Tag destination for a Service Tag Outbound Rule for the managed network of a machine learning workspace.
 
-    Variables are only populated by the server, and will be ignored when sending a request.
-
     :ivar action: The action enum for networking rule. Possible values include: "Allow", "Deny".
     :vartype action: str or ~azure.mgmt.machinelearningservices.models.RuleAction
     :ivar address_prefixes: Optional, if provided, the ServiceTag property will be ignored.
@@ -26602,10 +26600,6 @@ class ServiceTagDestination(msrest.serialization.Model):
     :vartype service_tag: str
     """
 
-    _validation = {
-        'address_prefixes': {'readonly': True},
-    }
-
     _attribute_map = {
         'action': {'key': 'action', 'type': 'str'},
         'address_prefixes': {'key': 'addressPrefixes', 'type': '[str]'},
@@ -26621,6 +26615,8 @@ def __init__(
         """
         :keyword action: The action enum for networking rule. Possible values include: "Allow", "Deny".
         :paramtype action: str or ~azure.mgmt.machinelearningservices.models.RuleAction
+        :keyword address_prefixes: Optional, if provided, the ServiceTag property will be ignored.
+        :paramtype address_prefixes: list[str]
         :keyword port_ranges:
         :paramtype port_ranges: str
         :keyword protocol:
@@ -26630,7 +26626,7 @@ def __init__(
         """
         super(ServiceTagDestination, self).__init__(**kwargs)
         self.action = kwargs.get('action', None)
-        self.address_prefixes = None
+        self.address_prefixes = kwargs.get('address_prefixes', None)
         self.port_ranges = kwargs.get('port_ranges', None)
         self.protocol = kwargs.get('protocol', None)
         self.service_tag = kwargs.get('service_tag', None)
@@ -31194,6 +31190,9 @@ class WorkspaceUpdateParameters(msrest.serialization.Model):
      ~azure.mgmt.machinelearningservices.models.ServiceManagedResourcesSettings
     :ivar soft_delete_retention_in_days: Retention time in days after workspace get soft deleted.
     :vartype soft_delete_retention_in_days: int
+    :ivar system_datastores_auth_mode: The auth mode used for accessing the system datastores of
+     the workspace.
+    :vartype system_datastores_auth_mode: str
     :ivar v1_legacy_mode: Enabling v1_legacy_mode may prevent you from using features provided by
      the v2 API.
     :vartype v1_legacy_mode: bool
@@ -31220,6 +31219,7 @@ class WorkspaceUpdateParameters(msrest.serialization.Model):
         'serverless_compute_settings': {'key': 'properties.serverlessComputeSettings', 'type': 'ServerlessComputeSettings'},
         'service_managed_resources_settings': {'key': 'properties.serviceManagedResourcesSettings', 'type': 'ServiceManagedResourcesSettings'},
         'soft_delete_retention_in_days': {'key': 'properties.softDeleteRetentionInDays', 'type': 'int'},
+        'system_datastores_auth_mode': {'key': 'properties.systemDatastoresAuthMode', 'type': 'str'},
         'v1_legacy_mode': {'key': 'properties.v1LegacyMode', 'type': 'bool'},
     }
 
@@ -31278,6 +31278,9 @@ def __init__(
         :keyword soft_delete_retention_in_days: Retention time in days after workspace get soft
          deleted.
         :paramtype soft_delete_retention_in_days: int
+        :keyword system_datastores_auth_mode: The auth mode used for accessing the system datastores of
+         the workspace.
+        :paramtype system_datastores_auth_mode: str
         :keyword v1_legacy_mode: Enabling v1_legacy_mode may prevent you from using features provided
          by the v2 API.
         :paramtype v1_legacy_mode: bool
@@ -31303,4 +31306,5 @@ def __init__(
         self.serverless_compute_settings = kwargs.get('serverless_compute_settings', None)
         self.service_managed_resources_settings = kwargs.get('service_managed_resources_settings', None)
         self.soft_delete_retention_in_days = kwargs.get('soft_delete_retention_in_days', None)
+        self.system_datastores_auth_mode = kwargs.get('system_datastores_auth_mode', None)
         self.v1_legacy_mode = kwargs.get('v1_legacy_mode', None)

sdk/ml/azure-ai-ml/azure/ai/ml/_restclient/v2024_07_01_preview/models/_models_py3.py

@@ -28837,8 +28837,6 @@ def __init__(
 class ServiceTagDestination(msrest.serialization.Model):
     """Service Tag destination for a Service Tag Outbound Rule for the managed network of a machine learning workspace.
 
-    Variables are only populated by the server, and will be ignored when sending a request.
-
     :ivar action: The action enum for networking rule. Possible values include: "Allow", "Deny".
     :vartype action: str or ~azure.mgmt.machinelearningservices.models.RuleAction
     :ivar address_prefixes: Optional, if provided, the ServiceTag property will be ignored.
@@ -28851,10 +28849,6 @@ class ServiceTagDestination(msrest.serialization.Model):
     :vartype service_tag: str
     """
 
-    _validation = {
-        'address_prefixes': {'readonly': True},
-    }
-
     _attribute_map = {
         'action': {'key': 'action', 'type': 'str'},
         'address_prefixes': {'key': 'addressPrefixes', 'type': '[str]'},
@@ -28867,6 +28861,7 @@ def __init__(
         self,
         *,
         action: Optional[Union[str, "RuleAction"]] = None,
+        address_prefixes: Optional[List[str]] = None,
         port_ranges: Optional[str] = None,
         protocol: Optional[str] = None,
         service_tag: Optional[str] = None,
@@ -28875,6 +28870,8 @@ def __init__(
         """
         :keyword action: The action enum for networking rule. Possible values include: "Allow", "Deny".
         :paramtype action: str or ~azure.mgmt.machinelearningservices.models.RuleAction
+        :keyword address_prefixes: Optional, if provided, the ServiceTag property will be ignored.
+        :paramtype address_prefixes: list[str]
         :keyword port_ranges:
         :paramtype port_ranges: str
         :keyword protocol:
@@ -28884,7 +28881,7 @@ def __init__(
         """
         super(ServiceTagDestination, self).__init__(**kwargs)
         self.action = action
-        self.address_prefixes = None
+        self.address_prefixes = address_prefixes
         self.port_ranges = port_ranges
         self.protocol = protocol
         self.service_tag = service_tag
@@ -33855,6 +33852,9 @@ class WorkspaceUpdateParameters(msrest.serialization.Model):
      ~azure.mgmt.machinelearningservices.models.ServiceManagedResourcesSettings
     :ivar soft_delete_retention_in_days: Retention time in days after workspace get soft deleted.
     :vartype soft_delete_retention_in_days: int
+    :ivar system_datastores_auth_mode: The auth mode used for accessing the system datastores of
+     the workspace.
+    :vartype system_datastores_auth_mode: str
     :ivar v1_legacy_mode: Enabling v1_legacy_mode may prevent you from using features provided by
      the v2 API.
     :vartype v1_legacy_mode: bool
@@ -33881,6 +33881,7 @@ class WorkspaceUpdateParameters(msrest.serialization.Model):
         'serverless_compute_settings': {'key': 'properties.serverlessComputeSettings', 'type': 'ServerlessComputeSettings'},
         'service_managed_resources_settings': {'key': 'properties.serviceManagedResourcesSettings', 'type': 'ServiceManagedResourcesSettings'},
         'soft_delete_retention_in_days': {'key': 'properties.softDeleteRetentionInDays', 'type': 'int'},
+        'system_datastores_auth_mode': {'key': 'properties.systemDatastoresAuthMode', 'type': 'str'},
         'v1_legacy_mode': {'key': 'properties.v1LegacyMode', 'type': 'bool'},
     }
 
@@ -33907,6 +33908,7 @@ def __init__(
         serverless_compute_settings: Optional["ServerlessComputeSettings"] = None,
         service_managed_resources_settings: Optional["ServiceManagedResourcesSettings"] = None,
         soft_delete_retention_in_days: Optional[int] = None,
+        system_datastores_auth_mode: Optional[str] = None,
         v1_legacy_mode: Optional[bool] = None,
         **kwargs
     ):
@@ -33961,6 +33963,9 @@ def __init__(
         :keyword soft_delete_retention_in_days: Retention time in days after workspace get soft
          deleted.
         :paramtype soft_delete_retention_in_days: int
+        :keyword system_datastores_auth_mode: The auth mode used for accessing the system datastores of
+         the workspace.
+        :paramtype system_datastores_auth_mode: str
         :keyword v1_legacy_mode: Enabling v1_legacy_mode may prevent you from using features provided
          by the v2 API.
         :paramtype v1_legacy_mode: bool
@@ -33986,4 +33991,5 @@ def __init__(
         self.serverless_compute_settings = serverless_compute_settings
         self.service_managed_resources_settings = service_managed_resources_settings
         self.soft_delete_retention_in_days = soft_delete_retention_in_days
+        self.system_datastores_auth_mode = system_datastores_auth_mode
         self.v1_legacy_mode = v1_legacy_mode

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_feature_store/feature_store_schema.py

@@ -11,7 +11,6 @@
 from azure.ai.ml._schema.workspace.identity import IdentitySchema, UserAssignedIdentitySchema
 from azure.ai.ml._utils.utils import snake_to_pascal
 from azure.ai.ml.constants._common import PublicNetworkAccess
-from azure.ai.ml._schema import ExperimentalField
 from azure.ai.ml._schema.workspace.networking import ManagedNetworkSchema
 from .compute_runtime_schema import ComputeRuntimeSchema
 from .materialization_store_schema import MaterializationStoreSchema
@@ -41,4 +40,4 @@ class FeatureStoreSchema(PathAwareSchema):
     )
     identity = NestedField(IdentitySchema)
     primary_user_assigned_identity = fields.Str()
-    managed_network = ExperimentalField(NestedField(ManagedNetworkSchema, unknown=EXCLUDE))
+    managed_network = NestedField(ManagedNetworkSchema, unknown=EXCLUDE)

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/networking.py

@@ -25,6 +25,7 @@ class ManagedNetworkStatusSchema(metaclass=PatchedSchemaMeta):
 
 class FqdnOutboundRuleSchema(metaclass=PatchedSchemaMeta):
     name = fields.Str(required=True)
+    parent_rule_names = fields.List(fields.Str(), dump_only=True)
     type = fields.Constant("fqdn")
     destination = fields.Str(required=True)
     category = StringTransformedEnum(
@@ -52,10 +53,12 @@ class ServiceTagDestinationSchema(metaclass=PatchedSchemaMeta):
     service_tag = fields.Str(required=True)
     protocol = fields.Str(required=True)
     port_ranges = fields.Str(required=True)
+    address_prefixes = fields.List(fields.Str())
 
 
 class ServiceTagOutboundRuleSchema(metaclass=PatchedSchemaMeta):
     name = fields.Str(required=True)
+    parent_rule_names = fields.List(fields.Str(), dump_only=True)
     type = fields.Constant("service_tag")
     destination = NestedField(ServiceTagDestinationSchema, required=True)
     category = StringTransformedEnum(
@@ -72,7 +75,9 @@ class ServiceTagOutboundRuleSchema(metaclass=PatchedSchemaMeta):
 
     @pre_dump
     def predump(self, data, **kwargs):
-        data.destination = self.service_tag_dest2dict(data.service_tag, data.protocol, data.port_ranges)
+        data.destination = self.service_tag_dest2dict(
+            data.service_tag, data.protocol, data.port_ranges, data.address_prefixes
+        )
         return data
 
     @post_load
@@ -86,15 +91,17 @@ def createdestobject(self, data, **kwargs):
             service_tag=dest["service_tag"],
             protocol=dest["protocol"],
             port_ranges=dest["port_ranges"],
+            address_prefixes=dest.get("address_prefixes", None),
             category=_snake_to_camel(category),
             status=status,
         )
 
-    def service_tag_dest2dict(self, service_tag, protocol, port_ranges):
+    def service_tag_dest2dict(self, service_tag, protocol, port_ranges, address_prefixes):
         service_tag_dest = {}
         service_tag_dest["service_tag"] = service_tag
         service_tag_dest["protocol"] = protocol
         service_tag_dest["port_ranges"] = port_ranges
+        service_tag_dest["address_prefixes"] = address_prefixes
         return service_tag_dest
 
 
@@ -106,8 +113,10 @@ class PrivateEndpointDestinationSchema(metaclass=PatchedSchemaMeta):
 
 class PrivateEndpointOutboundRuleSchema(metaclass=PatchedSchemaMeta):
     name = fields.Str(required=True)
+    parent_rule_names = fields.List(fields.Str(), dump_only=True)
     type = fields.Constant("private_endpoint")
     destination = NestedField(PrivateEndpointDestinationSchema, required=True)
+    fqdns = fields.List(fields.Str())
     category = StringTransformedEnum(
         allowed_values=[
             OutboundRuleCategory.REQUIRED,
@@ -132,13 +141,15 @@ def createdestobject(self, data, **kwargs):
         category = data.get("category", OutboundRuleCategory.USER_DEFINED)
         name = data.get("name")
         status = data.get("status", None)
+        fqdns = data.get("fqdns", None)
         return PrivateEndpointDestination(
             name=name,
             service_resource_id=dest["service_resource_id"],
             subresource_target=dest["subresource_target"],
             spark_enabled=dest["spark_enabled"],
             category=_snake_to_camel(category),
             status=status,
+            fqdns=fqdns,
         )
 
     def pe_dest2dict(self, service_resource_id, subresource_target, spark_enabled):

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/workspace.py

@@ -4,7 +4,6 @@
 
 from marshmallow import EXCLUDE, fields
 
-from azure.ai.ml._schema import ExperimentalField
 from azure.ai.ml._schema._utils.utils import validate_arm_str
 from azure.ai.ml._schema.core.fields import NestedField, StringTransformedEnum
 from azure.ai.ml._schema.core.schema import PathAwareSchema
@@ -41,7 +40,7 @@ class WorkspaceSchema(PathAwareSchema):
     identity = NestedField(IdentitySchema)
     primary_user_assigned_identity = fields.Str()
     workspace_hub = fields.Str(validate=validate_arm_str)
-    managed_network = ExperimentalField(NestedField(ManagedNetworkSchema, unknown=EXCLUDE))
+    managed_network = NestedField(ManagedNetworkSchema, unknown=EXCLUDE)
     enable_data_isolation = fields.Bool()
     allow_roleassignment_on_rg = fields.Bool()
     serverless_compute = NestedField(ServerlessComputeSettingsSchema)

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_feature_store/feature_store.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 from typing import Any, Dict, Optional, Union
 
-from azure.ai.ml._restclient.v2023_08_01_preview.models import Workspace as RestWorkspace
+from azure.ai.ml._restclient.v2024_07_01_preview.models import Workspace as RestWorkspace
 from azure.ai.ml._schema._feature_store.feature_store_schema import FeatureStoreSchema
 from azure.ai.ml.constants._common import BASE_PATH_CONTEXT_KEY, PARAMS_OVERRIDE_KEY
 from azure.ai.ml.entities._credentials import IdentityConfiguration, ManagedIdentityConfiguration

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_workspace/_ai_workspaces/hub.py

@@ -5,8 +5,10 @@
 # pylint: disable=too-many-instance-attributes,protected-access
 from typing import Any, Dict, List, Optional
 
-from azure.ai.ml._restclient.v2023_08_01_preview.models import Workspace as RestWorkspace
-from azure.ai.ml._restclient.v2023_08_01_preview.models import WorkspaceHubConfig as RestWorkspaceHubConfig
+from azure.ai.ml._restclient.v2024_07_01_preview.models import (
+    Workspace as RestWorkspace,
+    WorkspaceHubConfig as RestWorkspaceHubConfig,
+)
 from azure.ai.ml._schema.workspace import HubSchema
 from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml.constants._common import WorkspaceKind

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_workspace/diagnose.py

@@ -5,15 +5,11 @@
 import json
 from typing import Any, Dict, Optional, List
 
-from azure.ai.ml._restclient.v2023_08_01_preview.models import (
+from azure.ai.ml._restclient.v2024_07_01_preview.models import (
     DiagnoseRequestProperties as RestDiagnoseRequestProperties,
-)
-from azure.ai.ml._restclient.v2023_08_01_preview.models import DiagnoseResponseResult as RestDiagnoseResponseResult
-from azure.ai.ml._restclient.v2023_08_01_preview.models import (
+    DiagnoseResponseResult as RestDiagnoseResponseResult,
     DiagnoseResponseResultValue as RestDiagnoseResponseResultValue,
-)
-from azure.ai.ml._restclient.v2023_08_01_preview.models import DiagnoseResult as RestDiagnoseResult
-from azure.ai.ml._restclient.v2023_08_01_preview.models import (
+    DiagnoseResult as RestDiagnoseResult,
     DiagnoseWorkspaceParameters as RestDiagnoseWorkspaceParameters,
 )