Move remaining UCX integration tests and fixtures (#45)

Author: nfx

README.md

@@ -9,6 +9,7 @@
   * [Ecosystem](#ecosystem)
   * [PyTest Fixtures](#pytest-fixtures)
     * [Logging](#logging)
+    * [Installation](#installation)
     * [`debug_env_name` fixture](#debug_env_name-fixture)
     * [`debug_env` fixture](#debug_env-fixture)
     * [`env_or_skip` fixture](#env_or_skip-fixture)
@@ -74,6 +75,41 @@ also install it directly from the command line:
 pip install databricks-labs-pytester
 ```
 
+If you use `hatch` as a build system, make sure to add `databricks-labs-pytester` as 
+a test-time dependency and not as a compile-time dependency, otherwise your wheels will
+transitively depend on `pytest`, which is not usually something you need.
+
+```toml
+[project]
+name = "name-of-your-project"
+# ...
+dependencies = [
+  "databricks-sdk~=0.30",
+  # ... dependencies required for your code to execute
+]
+
+[tool.hatch.envs.default]
+dependencies = [
+  # ... dependencies required to test/validate/format your code:
+    "black~=24.3.0",
+    "coverage[toml]~=7.4.4",
+    "mypy~=1.9.0",
+    "pylint~=3.2.2",
+    "pylint-pytest==2.0.0a0",
+    "databricks-labs-pylint~=0.4.0",
+    "databricks-labs-pytester~=0.2", # <= this library 
+    "pytest~=8.3.3",
+    "pytest-cov~=4.1.0",
+    "pytest-mock~=3.14.0",
+    "pytest-timeout~=2.3.1",
+    "pytest-xdist~=3.5.0",
+    "python-lsp-server>=1.9.0",
+    "ruff~=0.3.4",
+    "types-PyYAML~=6.0.12",
+    "types-requests~=2.31.0",
+]
+```
+
 [[back to top](#python-testing-for-databricks)]
 
 ## Ecosystem
@@ -242,7 +278,7 @@ def test_workspace_operations(ws):
     assert len(clusters) >= 0
 ```
 
-See also [`log_workspace_link`](#log_workspace_link-fixture), [`make_alert_permissions`](#make_alert_permissions-fixture), [`make_authorization_permissions`](#make_authorization_permissions-fixture), [`make_catalog`](#make_catalog-fixture), [`make_cluster`](#make_cluster-fixture), [`make_cluster_permissions`](#make_cluster_permissions-fixture), [`make_cluster_policy`](#make_cluster_policy-fixture), [`make_cluster_policy_permissions`](#make_cluster_policy_permissions-fixture), [`make_dashboard_permissions`](#make_dashboard_permissions-fixture), [`make_directory`](#make_directory-fixture), [`make_directory_permissions`](#make_directory_permissions-fixture), [`make_experiment`](#make_experiment-fixture), [`make_experiment_permissions`](#make_experiment_permissions-fixture), [`make_feature_table_permissions`](#make_feature_table_permissions-fixture), [`make_group`](#make_group-fixture), [`make_instance_pool`](#make_instance_pool-fixture), [`make_instance_pool_permissions`](#make_instance_pool_permissions-fixture), [`make_job`](#make_job-fixture), [`make_job_permissions`](#make_job_permissions-fixture), [`make_lakeview_dashboard_permissions`](#make_lakeview_dashboard_permissions-fixture), [`make_model`](#make_model-fixture), [`make_notebook`](#make_notebook-fixture), [`make_notebook_permissions`](#make_notebook_permissions-fixture), [`make_pipeline`](#make_pipeline-fixture), [`make_pipeline_permissions`](#make_pipeline_permissions-fixture), [`make_query`](#make_query-fixture), [`make_query_permissions`](#make_query_permissions-fixture), [`make_registered_model_permissions`](#make_registered_model_permissions-fixture), [`make_repo`](#make_repo-fixture), [`make_repo_permissions`](#make_repo_permissions-fixture), [`make_secret_scope`](#make_secret_scope-fixture), [`make_secret_scope_acl`](#make_secret_scope_acl-fixture), [`make_serving_endpoint`](#make_serving_endpoint-fixture), [`make_serving_endpoint_permissions`](#make_serving_endpoint_permissions-fixture), [`make_storage_credential`](#make_storage_credential-fixture), [`make_udf`](#make_udf-fixture), [`make_user`](#make_user-fixture), [`make_warehouse`](#make_warehouse-fixture), [`make_warehouse_permissions`](#make_warehouse_permissions-fixture), [`make_workspace_file_path_permissions`](#make_workspace_file_path_permissions-fixture), [`make_workspace_file_permissions`](#make_workspace_file_permissions-fixture), [`spark`](#spark-fixture), [`sql_backend`](#sql_backend-fixture), [`debug_env`](#debug_env-fixture), [`product_info`](#product_info-fixture).
+See also [`log_workspace_link`](#log_workspace_link-fixture), [`make_alert_permissions`](#make_alert_permissions-fixture), [`make_authorization_permissions`](#make_authorization_permissions-fixture), [`make_catalog`](#make_catalog-fixture), [`make_cluster`](#make_cluster-fixture), [`make_cluster_permissions`](#make_cluster_permissions-fixture), [`make_cluster_policy`](#make_cluster_policy-fixture), [`make_cluster_policy_permissions`](#make_cluster_policy_permissions-fixture), [`make_dashboard_permissions`](#make_dashboard_permissions-fixture), [`make_directory`](#make_directory-fixture), [`make_directory_permissions`](#make_directory_permissions-fixture), [`make_experiment`](#make_experiment-fixture), [`make_experiment_permissions`](#make_experiment_permissions-fixture), [`make_feature_table`](#make_feature_table-fixture), [`make_feature_table_permissions`](#make_feature_table_permissions-fixture), [`make_group`](#make_group-fixture), [`make_instance_pool`](#make_instance_pool-fixture), [`make_instance_pool_permissions`](#make_instance_pool_permissions-fixture), [`make_job`](#make_job-fixture), [`make_job_permissions`](#make_job_permissions-fixture), [`make_lakeview_dashboard_permissions`](#make_lakeview_dashboard_permissions-fixture), [`make_model`](#make_model-fixture), [`make_notebook`](#make_notebook-fixture), [`make_notebook_permissions`](#make_notebook_permissions-fixture), [`make_pipeline`](#make_pipeline-fixture), [`make_pipeline_permissions`](#make_pipeline_permissions-fixture), [`make_query`](#make_query-fixture), [`make_query_permissions`](#make_query_permissions-fixture), [`make_registered_model_permissions`](#make_registered_model_permissions-fixture), [`make_repo`](#make_repo-fixture), [`make_repo_permissions`](#make_repo_permissions-fixture), [`make_secret_scope`](#make_secret_scope-fixture), [`make_secret_scope_acl`](#make_secret_scope_acl-fixture), [`make_serving_endpoint`](#make_serving_endpoint-fixture), [`make_serving_endpoint_permissions`](#make_serving_endpoint_permissions-fixture), [`make_storage_credential`](#make_storage_credential-fixture), [`make_udf`](#make_udf-fixture), [`make_user`](#make_user-fixture), [`make_warehouse`](#make_warehouse-fixture), [`make_warehouse_permissions`](#make_warehouse_permissions-fixture), [`make_workspace_file_path_permissions`](#make_workspace_file_path_permissions-fixture), [`make_workspace_file_permissions`](#make_workspace_file_permissions-fixture), [`spark`](#spark-fixture), [`sql_backend`](#sql_backend-fixture), [`debug_env`](#debug_env-fixture), [`product_info`](#product_info-fixture).
 
 
 [[back to top](#python-testing-for-databricks)]
@@ -336,7 +372,7 @@ random_string = make_random(k=8)
 assert len(random_string) == 8
 ```
 
-See also [`make_acc_group`](#make_acc_group-fixture), [`make_catalog`](#make_catalog-fixture), [`make_cluster`](#make_cluster-fixture), [`make_cluster_policy`](#make_cluster_policy-fixture), [`make_directory`](#make_directory-fixture), [`make_experiment`](#make_experiment-fixture), [`make_group`](#make_group-fixture), [`make_instance_pool`](#make_instance_pool-fixture), [`make_job`](#make_job-fixture), [`make_model`](#make_model-fixture), [`make_notebook`](#make_notebook-fixture), [`make_pipeline`](#make_pipeline-fixture), [`make_query`](#make_query-fixture), [`make_repo`](#make_repo-fixture), [`make_schema`](#make_schema-fixture), [`make_secret_scope`](#make_secret_scope-fixture), [`make_serving_endpoint`](#make_serving_endpoint-fixture), [`make_table`](#make_table-fixture), [`make_udf`](#make_udf-fixture), [`make_user`](#make_user-fixture), [`make_warehouse`](#make_warehouse-fixture).
+See also [`make_acc_group`](#make_acc_group-fixture), [`make_catalog`](#make_catalog-fixture), [`make_cluster`](#make_cluster-fixture), [`make_cluster_policy`](#make_cluster_policy-fixture), [`make_directory`](#make_directory-fixture), [`make_experiment`](#make_experiment-fixture), [`make_feature_table`](#make_feature_table-fixture), [`make_group`](#make_group-fixture), [`make_instance_pool`](#make_instance_pool-fixture), [`make_job`](#make_job-fixture), [`make_model`](#make_model-fixture), [`make_notebook`](#make_notebook-fixture), [`make_pipeline`](#make_pipeline-fixture), [`make_query`](#make_query-fixture), [`make_repo`](#make_repo-fixture), [`make_schema`](#make_schema-fixture), [`make_secret_scope`](#make_secret_scope-fixture), [`make_serving_endpoint`](#make_serving_endpoint-fixture), [`make_table`](#make_table-fixture), [`make_udf`](#make_udf-fixture), [`make_user`](#make_user-fixture), [`make_warehouse`](#make_warehouse-fixture).
 
 
 [[back to top](#python-testing-for-databricks)]
@@ -370,7 +406,8 @@ See also [`ws`](#ws-fixture).
 [[back to top](#python-testing-for-databricks)]
 
 ### `make_job` fixture
-Create a Databricks job and clean it up after the test. Returns a function to create jobs.
+Create a Databricks job and clean it up after the test. Returns a function to create jobs, that returns
+a [`Job`](https://databricks-sdk-py.readthedocs.io/en/latest/dbdataclasses/jobs.html#databricks.sdk.service.jobs.Job) instance.
 
 Keyword Arguments:
 * `notebook_path` (str, optional): The path to the notebook. If not provided, a random notebook will be created.
@@ -1046,6 +1083,14 @@ _No description yet._
 See also [`ws`](#ws-fixture).
 
 
+[[back to top](#python-testing-for-databricks)]
+
+### `make_feature_table` fixture
+_No description yet._
+
+See also [`ws`](#ws-fixture), [`make_random`](#make_random-fixture).
+
+
 [[back to top](#python-testing-for-databricks)]
 
 ### `make_feature_table_permissions` fixture

pyproject.toml

@@ -88,7 +88,7 @@ coverage    = "pytest -n auto --cov src tests/unit --timeout 30 --cov-report=htm
 integration = "pytest -n 10 --cov src tests/integration --durations 20"
 fmt         = ["black . --extend-exclude 'tests/unit/source_code/samples/'",
                "ruff check . --fix",
-               "mypy --disable-error-code 'annotation-unchecked' --enable-incomplete-feature=NewGenericSyntax --exclude 'tests/resources/*' --exclude dist .",
+               "mypy --disable-error-code 'annotation-unchecked' --disable-error-code import-untyped --enable-incomplete-feature=NewGenericSyntax --exclude 'tests/resources/*' --exclude dist .",
                "pylint --output-format=colorized -j 0 src tests"]
 verify      = ["black --check . --extend-exclude 'tests/unit/source_code/samples/'",
                "ruff check .",

src/databricks/labs/pytester/fixtures/compute.py

@@ -157,7 +157,8 @@ def create(*, instance_pool_name=None, node_type_id=None, **kwargs) -> CreateIns
 @fixture
 def make_job(ws, make_random, make_notebook, log_workspace_link, watchdog_remove_after) -> Generator[Job, None, None]:
     """
-    Create a Databricks job and clean it up after the test. Returns a function to create jobs.
+    Create a Databricks job and clean it up after the test. Returns a function to create jobs, that returns
+    a `databricks.sdk.service.jobs.Job` instance.
 
     Keyword Arguments:
     * `notebook_path` (str, optional): The path to the notebook. If not provided, a random notebook will be created.
@@ -211,7 +212,7 @@ def create(notebook_path: str | Path | None = None, **kwargs) -> Job:
             kwargs["tags"].append(remove_after_tag)
         job = ws.jobs.create(**kwargs)
         log_workspace_link(kwargs["name"], f'job/{job.job_id}', anchor=False)
-        return job
+        return ws.jobs.get(job.job_id)
 
     yield from factory("job", create, lambda item: ws.jobs.delete(item.job_id))
 

src/databricks/labs/pytester/fixtures/ml.py

@@ -145,3 +145,20 @@ def remove(endpoint_name: str):
         ws.serving_endpoints.delete(endpoint_name)
 
     yield from factory("Serving endpoint", create, remove)
+
+
+@fixture
+def make_feature_table(ws, make_random):
+    def create():
+        feature_table_name = make_random(6) + "." + make_random(6)
+        table = ws.api_client.do(
+            "POST",
+            "/api/2.0/feature-store/feature-tables/create",
+            body={"name": feature_table_name, "primary_keys": [{"name": "pk", "data_type": "string"}]},
+        )
+        return table['feature_table']
+
+    def remove(table: dict):
+        ws.api_client.do("DELETE", "/api/2.0/feature-store/feature-tables/delete", body={"name": table["name"]})
+
+    yield from factory("Feature table", create, remove)

src/databricks/labs/pytester/fixtures/plugin.py

@@ -49,7 +49,7 @@
 )
 from databricks.labs.pytester.fixtures.secrets import make_secret_scope, make_secret_scope_acl
 from databricks.labs.pytester.fixtures.environment import debug_env, debug_env_name, env_or_skip, is_in_debug
-from databricks.labs.pytester.fixtures.ml import make_experiment, make_model, make_serving_endpoint
+from databricks.labs.pytester.fixtures.ml import make_experiment, make_model, make_serving_endpoint, make_feature_table
 from databricks.labs.pytester.fixtures.redash import make_query
 from databricks.labs.pytester.fixtures.watchdog import watchdog_remove_after, watchdog_purge_suffix
 from databricks.labs.pytester.fixtures.connect import spark
@@ -109,6 +109,7 @@
     'make_registered_model_permissions',
     'make_serving_endpoint',
     'make_serving_endpoint_permissions',
+    'make_feature_table',
     'make_feature_table_permissions',
     'watchdog_remove_after',
     'watchdog_purge_suffix',

tests/integration/fixtures/test_catalog.py

@@ -2,9 +2,6 @@
 import pytest
 
 
-
-
-
 logger = logging.getLogger(__name__)