Update SDK to Support Serverless Custom VNet on Workspace Create and Update (Azure#32520)

* Added support for creating workspace with Serverless custom VNET and updating the settings of it

* Removing Protocol (added in 3.8) to support Python 3.7

* Fixed tests

* Updated Diagnosis deserialization to also account for updated API version

* Addressing review comments and failing gates. Also bumping guidelines version to match global CI version

---------

Co-authored-by: Sam Rhody <[email protected]>

Author: Hittherhod

sdk/ml/azure-ai-ml/.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
         dependencies = [
             (
                 "azure-pylint-guidelines-checker",
-                "0.1.0",
+                "0.2.0",
                 [
                     "--index-url",
                     "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/",

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -20,6 +20,7 @@
 ### Features Added
 - Now, when you specify `delete_dependent_resources` as True when deleting a workspace, the log analytics resource
   associated with the workspace application insights resource will also be deleted.
+- Now, when creating or updating a workspace, you can provide a `serverless_compute` configuration object. This allows configuring a custom subnet in which all Serverless computes will be created. You can also specify whether or not these Serverless computes will have public IP addresses or not.
 
 ### Breaking Changes
  - [Python 3.7 reached end-of-life on June 27th 2023](https://devguide.python.org/versions/). Consequently, 3.7 will be deprecated in azure-ai-ml starting in October 2023 and azure-ai-ml will end support for 3.7 in February 2024.

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json

@@ -553,6 +553,13 @@
             "metadata": {
                 "description": "Resource Id of BYO open ai resource."
             }
+        },
+        "serverless_compute_settings": {
+            "type": "object",
+            "defaultValue": {},
+            "metadata": {
+                "description": "Serverless compute settings to be used for the workspace."
+            }
         }
     },
     "variables": {
@@ -750,7 +757,7 @@
         {
             "condition": "[variables('enablePE')]",
             "type": "Microsoft.MachineLearningServices/workspaces",
-            "apiVersion": "2023-04-01-preview",
+            "apiVersion": "2023-08-01-preview",
             "tags": "[parameters('tagValues')]",
             "name": "[parameters('workspaceName')]",
             "kind": "[parameters('kind')]",
@@ -798,7 +805,8 @@
                 "existingWorkspaces": "[parameters('existing_workspaces')]",
                 "azureOpenAI": {
                     "resourceId": "[parameters('byo_open_ai_resource_id')]"
-                }
+                },
+                "serverlessComputeSettings": "[parameters('serverless_compute_settings')]"
             }
         },
         {

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_param.json

@@ -181,5 +181,8 @@
     },
     "byo_open_ai_resource_id": {
         "value": ""
+    },
+    "serverless_compute_settings": {
+        "value": {}
     }
 }

sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_project.json

@@ -592,6 +592,13 @@
             "metadata": {
                 "description": "Resource Id of BYO open ai resource."
             }
+        },
+        "serverless_compute_settings": {
+            "type": "object",
+            "defaultValue": {},
+            "metadata": {
+                "description": "Serverless compute configuration."
+            }
         }
     },
     "variables": {
@@ -663,7 +670,7 @@
         {
             "condition": "[variables('enablePE')]",
             "type": "Microsoft.MachineLearningServices/workspaces",
-            "apiVersion": "2023-04-01-preview",
+            "apiVersion": "2023-08-01-preview",
             "tags": "[parameters('tagValues')]",
             "name": "[parameters('workspaceName')]",
             "kind": "[parameters('kind')]",
@@ -680,7 +687,8 @@
                 "hubResourceId": "[parameters('workspace_hub')]",
                 "azureOpenAI": {
                     "resourceId": "[parameters('byo_open_ai_resource_id')]"
-                }
+                },
+                "serverlessComputeSettings": "[parameters('serverless_compute_settings')]"
             }
         }
     ]

sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py

@@ -399,7 +399,7 @@ def __init__(
 
         self._workspaces = WorkspaceOperations(
             self._ws_operation_scope if registry_reference else self._operation_scope,
-            self._service_client_06_2023_preview,
+            self._service_client_08_2023_preview,
             self._operation_container,
             self._credential,
             **app_insights_handler_kwargs,

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/serverless_compute.py

@@ -0,0 +1,35 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from marshmallow import fields
+from marshmallow.decorators import post_load
+
+from azure.ai.ml._schema._utils.utils import validate_arm_str
+from azure.ai.ml._schema.core.schema import PathAwareSchema
+from azure.ai.ml.entities._workspace.serverless_compute import ServerlessComputeSettings
+
+
+class ServerlessComputeSettingsSchema(PathAwareSchema):
+    """Schema for ServerlessComputeSettings.
+
+    :param custom_subnet: The custom subnet to use for serverless computes created in the workspace.
+    :type custom_subnet: str. Formatted as ARM ID.
+    :param no_public_ip: Whether to disable public ip for the compute. Only valid if custom_subnet is defined.
+    :type no_public_ip: bool
+    """
+
+    custom_subnet = fields.Str(validate=validate_arm_str)
+    no_public_ip = fields.Bool(load_default=False)
+
+    @post_load
+    def make(self, data, **_kwargs) -> ServerlessComputeSettings:
+        """Create a ServerlessComputeSettings object from the marshmallow schema.
+
+        :param data: The data from which the ServerlessComputeSettings are being loaded.
+        :type data: OrderedDict[str, Any]
+        :returns: A ServerlessComputeSettings object.
+        :rtype: azure.ai.ml.entities._workspace.serverless_compute.ServerlessComputeSettings
+        """
+        custom_subnet = data.pop("custom_subnet")
+        no_public_ip = data.pop("no_public_ip")
+        return ServerlessComputeSettings(custom_subnet, no_public_ip)

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/workspace.py

@@ -2,18 +2,18 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from marshmallow import fields, EXCLUDE
+from marshmallow import EXCLUDE, fields
 
+from azure.ai.ml._schema import ExperimentalField
 from azure.ai.ml._schema._utils.utils import validate_arm_str
 from azure.ai.ml._schema.core.fields import NestedField, StringTransformedEnum
 from azure.ai.ml._schema.core.schema import PathAwareSchema
 from azure.ai.ml._schema.workspace.customer_managed_key import CustomerManagedKeySchema
 from azure.ai.ml._schema.workspace.identity import IdentitySchema
+from azure.ai.ml._schema.workspace.networking import ManagedNetworkSchema
+from azure.ai.ml._schema.workspace.serverless_compute import ServerlessComputeSettingsSchema
 from azure.ai.ml._utils.utils import snake_to_pascal
 from azure.ai.ml.constants._common import PublicNetworkAccess
-from azure.ai.ml._schema.workspace.networking import ManagedNetworkSchema
-
-from azure.ai.ml._schema import ExperimentalField
 
 
 class WorkspaceSchema(PathAwareSchema):
@@ -42,3 +42,4 @@ class WorkspaceSchema(PathAwareSchema):
     workspace_hub = fields.Str(validate=validate_arm_str)
     managed_network = ExperimentalField(NestedField(ManagedNetworkSchema, unknown=EXCLUDE))
     enable_data_isolation = fields.Bool()
+    serverless_compute = NestedField(ServerlessComputeSettingsSchema)