Add projects and hubs to AML SDK. (#35142)

* initial dump

* run black

* black, mypy, lint

* test corrections 1

* rerun black

* remove outdated test

* refactor polymorphism to not need more load functions

* syntax checks

* refactor constant to not be enum to avoid type problems

* update assets

* more test corrections

* more test corrections, feature store kind refactor

* rename type to kind

* show associated workspaces in yaml view

* assign project values from hub during creation

* formatting

* record pna in projects

* rename remove project from default_project_resource_group

* Update sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py

Co-authored-by: kdestin <[email protected]>

* Update sdk/ml/azure-ai-ml/azure/ai/ml/operations/_workspace_operations.py

Co-authored-by: kdestin <[email protected]>

* Update sdk/ml/azure-ai-ml/azure/ai/ml/operations/_workspace_operations.py

Co-authored-by: kdestin <[email protected]>

* Update sdk/ml/azure-ai-ml/azure/ai/ml/operations/_workspace_operations.py

Co-authored-by: kdestin <[email protected]>

* name consistency

* undo enumification of workspace kind

---------

Co-authored-by: kdestin <[email protected]>

Author: MilesHolland

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -4,10 +4,19 @@
 
 ### Features Added
 
+- Renamed WorkspaceHub class as Hub.
+- Added Project entity class and YAML support.
+- Project and Hub operations supported by workspace operations.
+- workspace list operation supports type filtering.
+
 ### Bugs Fixed
 
 ### Breaking Changes
 
+- Removed WorkspaceHubConfig entity, and renamed WorkspaceHub to Hub.
+- workspace_hub input of Workspace class hidden, renamed to hub_id, and re-surfaced in child class Project.
+- Removed Workspace Hub Operations from ML Client.
+
 ### Other Changes
 
 ## 1.15.0 (2024-03-26)

sdk/ml/azure-ai-ml/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/ml/azure-ai-ml",
-  "Tag": "python/ml/azure-ai-ml_baea630318"
+  "Tag": "python/ml/azure-ai-ml_cfba1d7771"
 }

sdk/ml/azure-ai-ml/azure/ai/ml/__init__.py

@@ -35,7 +35,6 @@
     load_registry,
     load_workspace,
     load_workspace_connection,
-    load_workspace_hub,
 )
 
 module_logger = logging.getLogger(__name__)
@@ -69,7 +68,6 @@
     "load_workspace",
     "load_registry",
     "load_workspace_connection",
-    "load_workspace_hub",
     "load_model_package",
 ]
 

sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py

@@ -43,7 +43,6 @@
 from azure.ai.ml._scope_dependent_operations import OperationConfig, OperationsContainer, OperationScope
 from azure.ai.ml._telemetry.logging_handler import get_appinsights_log_handler
 from azure.ai.ml._user_agent import USER_AGENT
-from azure.ai.ml._utils._experimental import experimental
 from azure.ai.ml._utils._http_utils import HttpPipeline
 from azure.ai.ml._utils._preflight_utils import get_deployments_operation
 from azure.ai.ml._utils._registry_utils import get_registry_client
@@ -82,7 +81,6 @@
     OnlineEndpointOperations,
     RegistryOperations,
     WorkspaceConnectionsOperations,
-    WorkspaceHubOperations,
     WorkspaceOperations,
 )
 from azure.ai.ml.operations._code_operations import CodeOperations
@@ -658,15 +656,6 @@ def __init__(
             **ops_kwargs,  # type: ignore[arg-type]
         )
 
-        self._workspace_hubs = WorkspaceHubOperations(
-            self._operation_scope,
-            self._service_client_08_2023_preview,
-            self._operation_container,
-            self._credential,
-            **app_insights_handler_kwargs,  # type: ignore[arg-type]
-        )
-        self._operation_container.add(AzureMLResourceType.WORKSPACE_HUB, self._workspace_hubs)  # type: ignore[arg-type]
-
         self._operation_container.add(AzureMLResourceType.FEATURE_STORE, self._featurestores)  # type: ignore[arg-type]
         self._operation_container.add(AzureMLResourceType.FEATURE_SET, self._featuresets)
         self._operation_container.add(AzureMLResourceType.FEATURE_STORE_ENTITY, self._featurestoreentities)
@@ -811,7 +800,8 @@ def _ml_client_cli(cls, credentials: TokenCredential, subscription_id: Optional[
 
     @property
     def workspaces(self) -> WorkspaceOperations:
-        """A collection of workspace-related operations.
+        """A collection of workspace-related operations. Also manages workspace
+        sub-classes like projects and hubs.
 
         :return: Workspace operations
         :rtype: ~azure.ai.ml.operations.WorkspaceOperations
@@ -854,16 +844,6 @@ def feature_sets(self) -> FeatureSetOperations:
         """
         return self._featuresets
 
-    @property
-    @experimental
-    def workspace_hubs(self) -> WorkspaceHubOperations:
-        """A collection of workspace hub-related operations.
-
-        :return: Hub Operations
-        :rtype: HubOperations
-        """
-        return self._workspace_hubs
-
     @property
     def feature_store_entities(self) -> FeatureStoreEntityOperations:
         """A collection of feature store entity related operations.

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_workspace_hub/workspace_hub.py

@@ -5,5 +5,7 @@
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
 
 from .workspace import WorkspaceSchema
+from .ai_workspaces.project import ProjectSchema
+from .ai_workspaces.hub import HubSchema
 
-__all__ = ["WorkspaceSchema"]
+__all__ = ["WorkspaceSchema", "ProjectSchema", "HubSchema"]

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_workspace_hub/workspace_hub_config.py

@@ -3,7 +3,3 @@
 # ---------------------------------------------------------
 
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
-
-from .workspace_hub import WorkspaceHubSchema
-
-__all__ = ["WorkspaceHubSchema"]

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/__init__.py

@@ -0,0 +1,18 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from marshmallow import fields
+
+from azure.ai.ml._schema import StringTransformedEnum
+from azure.ai.ml._schema.workspace import WorkspaceSchema
+from azure.ai.ml._utils._experimental import experimental
+from azure.ai.ml.constants import WorkspaceKind
+
+
+@experimental
+class HubSchema(WorkspaceSchema):
+    # additional_workspace_storage_accounts This field exists in the API, but is unused, and thus not surfaced yet.
+    kind = StringTransformedEnum(required=True, allowed_values=WorkspaceKind.HUB)
+    default_resource_group = fields.Str(required=False)
+    associated_workspaces = fields.List(fields.Str(), required=False, dump_only=True)

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_workspace_hub/__init__.py renamed to sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/ai_workspaces/__init__.py

@@ -0,0 +1,16 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from marshmallow import fields
+
+from azure.ai.ml._schema import StringTransformedEnum
+from azure.ai.ml._utils._experimental import experimental
+from azure.ai.ml._schema.workspace import WorkspaceSchema
+from azure.ai.ml.constants import WorkspaceKind
+
+
+@experimental
+class ProjectSchema(WorkspaceSchema):
+    kind = StringTransformedEnum(required=True, allowed_values=WorkspaceKind.PROJECT)
+    hub_id = fields.Str(required=True)