Organise modules by domain (#197)

This simplifies code and moves modules and classes by their respective
domains/components, roughly mapped on folders. This is a change from the
previous diamond-shaped cross-module dependency layout, which is harder
to evolve.

Main tenets be:

- any component can depend on `mixins` (staging are for SDK) and
`framework` (common modules).
- tight coupling is allowed within a single component. An example is
local group migration, which exposes a single high-level migration
toolkit (`from databricks.labs.ucx.workspace_access import
GroupMigrationToolkit`) and leaves implementation details private.
Another example is a toolkit to migrate from HMS to UC DDL.
- all execution is either triggered from `install` (which also handles
update) or `runtime` (as the entrypoint for jobs). Multiple toolkits may
be combined into a single Databricks Workflows (aka Jobs) - e.g. pulling
permissions and tables inventory is necessary for the assessment step,
but later the Workflows may branch out.
- if two classes are used together in 90% of cases, they have to be
defined in the single file (aka Python Module). We don't define `types`
and `managers` modules/packages, as this is an example of a
diamond-shaped dependencies, which are more difficult to evolve over
time.
- methods and fields have to be private by default. make them public
only by necessity.
- classes must have the following order - __init__, overridden methods,
public methods, other methods. the most important logic has to be first.

Author: nfx

notebooks/toolkit.py

@@ -17,8 +17,8 @@
     MigrationConfig,
     TaclConfig,
 )
-from databricks.labs.ucx.toolkits.group_migration import GroupMigrationToolkit
-from databricks.labs.ucx.toolkits.table_acls import TaclToolkit
+from databricks.labs.ucx.workspace_access import GroupMigrationToolkit
+from databricks.labs.ucx.hive_metastore import TaclToolkit
 
 # COMMAND ----------
 

pyproject.toml

@@ -163,14 +163,14 @@ ban-relative-imports = "all"
     "ARG001" # tests may not use the provided fixtures
 ]
 
-"src/databricks/labs/ucx/providers/mixins/redash.py" = ["A002", "A003", "N815"]
+"src/databricks/labs/ucx/mixins/redash.py" = ["A002", "A003", "N815"]
 
 [tool.coverage.run]
 branch = true
 parallel = true
 
 [tool.coverage.report]
-omit = ["src/databricks/labs/ucx/providers/mixins/*"]
+omit = ["src/databricks/labs/ucx/mixins/*"]
 exclude_lines = [
     "no cov",
     "if __name__ == .__main__.:",

src/databricks/labs/ucx/__init__.py

@@ -1,3 +1,3 @@
-from databricks.labs.ucx.logger import _install
+from databricks.labs.ucx.framework.logger import _install
 
 _install()

src/databricks/labs/ucx/toolkits/assessment.py renamed to src/databricks/labs/ucx/assessment/assessment.py

@@ -6,7 +6,7 @@
 from databricks.sdk.service.compute import Language
 
 from databricks.labs.ucx.assessment import commands
-from databricks.labs.ucx.providers.mixins.compute import CommandExecutor
+from databricks.labs.ucx.mixins.compute import CommandExecutor
 
 logger = logging.getLogger(__name__)
 

src/databricks/labs/ucx/managers/__init__.py renamed to src/databricks/labs/ucx/framework/__init__.py

@@ -6,7 +6,7 @@
 
 from databricks.sdk import WorkspaceClient
 
-from databricks.labs.ucx.providers.mixins.sql import StatementExecutionExt
+from databricks.labs.ucx.mixins.sql import StatementExecutionExt
 
 logger = logging.getLogger(__name__)
 

src/databricks/labs/ucx/tacl/_internal.py renamed to src/databricks/labs/ucx/framework/crawlers.py

@@ -5,8 +5,6 @@
 from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor
 from typing import Generic, TypeVar
 
-from databricks.labs.ucx.generic import StrEnum
-
 ExecutableResult = TypeVar("ExecutableResult")
 ExecutableFunction = Callable[..., ExecutableResult]
 logger = logging.getLogger(__name__)
@@ -66,20 +64,5 @@ def run(self) -> list[ExecutableResult]:
         return collected
 
 
-class Request:
-    def __init__(self, req: dict):
-        self.request = req
-
-    def as_dict(self) -> dict:
-        return self.request
-
-
-class WorkspaceLevelEntitlement(StrEnum):
-    WORKSPACE_ACCESS = "workspace-access"
-    DATABRICKS_SQL_ACCESS = "databricks-sql-access"
-    ALLOW_CLUSTER_CREATE = "allow-cluster-create"
-    ALLOW_INSTANCE_POOL_CREATE = "allow-instance-pool-create"
-
-
 def noop():
     pass

src/databricks/labs/ucx/logger.py renamed to src/databricks/labs/ucx/framework/logger.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 from databricks.labs.ucx.config import MigrationConfig
-from databricks.labs.ucx.logger import _install
+from databricks.labs.ucx.framework.logger import _install
 
 _TASKS: dict[str, "Task"] = {}