|
| 1 | +import base64 |
| 2 | +import json |
1 | 3 | import logging |
| 4 | +import pathlib |
| 5 | +from unittest.mock import create_autospec |
| 6 | + |
| 7 | +from databricks.sdk import WorkspaceClient |
| 8 | +from databricks.sdk.errors import NotFound |
| 9 | +from databricks.sdk.service.compute import ClusterDetails, Policy |
| 10 | +from databricks.sdk.service.jobs import BaseJob, BaseRun |
| 11 | +from databricks.sdk.service.pipelines import GetPipelineResponse, PipelineStateInfo |
| 12 | +from databricks.sdk.service.sql import EndpointConfPair |
| 13 | +from databricks.sdk.service.workspace import ExportResponse, GetSecretResponse |
| 14 | + |
| 15 | +from databricks.labs.ucx.hive_metastore.mapping import TableMapping, TableToMigrate |
2 | 16 |
|
3 | 17 | logging.getLogger("tests").setLevel("DEBUG") |
4 | 18 |
|
|
13 | 27 | }, |
14 | 28 | }, |
15 | 29 | } |
| 30 | + |
| 31 | +__dir = pathlib.Path(__file__).parent |
| 32 | + |
| 33 | + |
| 34 | +def _base64(filename: str): |
| 35 | + try: |
| 36 | + with (__dir / filename).open("rb") as f: |
| 37 | + return base64.b64encode(f.read()) |
| 38 | + except FileNotFoundError as err: |
| 39 | + raise NotFound(filename) from err |
| 40 | + |
| 41 | + |
| 42 | +def _workspace_export(filename: str): |
| 43 | + res = _base64(f'workspace/{filename}') |
| 44 | + return ExportResponse(content=res.decode('utf8')) |
| 45 | + |
| 46 | + |
| 47 | +def _load_fixture(filename: str): |
| 48 | + try: |
| 49 | + with (__dir / filename).open("r") as f: |
| 50 | + return json.load(f) |
| 51 | + except FileNotFoundError as err: |
| 52 | + raise NotFound(filename) from err |
| 53 | + |
| 54 | + |
| 55 | +_FOLDERS = { |
| 56 | + BaseJob: 'assessment/jobs', |
| 57 | + BaseRun: 'assessment/jobruns', |
| 58 | + ClusterDetails: 'assessment/clusters', |
| 59 | + PipelineStateInfo: 'assessment/pipelines', |
| 60 | + Policy: 'assessment/policies', |
| 61 | + TableToMigrate: 'hive_metastore/tables', |
| 62 | +} |
| 63 | + |
| 64 | + |
| 65 | +def _load_list(cls: type, filename: str, ids=None): |
| 66 | + if not ids: # TODO: remove |
| 67 | + return [cls.from_dict(_) for _ in _load_fixture(filename)] # type: ignore[attr-defined] |
| 68 | + return _id_list(cls, ids) |
| 69 | + |
| 70 | + |
| 71 | +def _id_list(cls: type, ids=None): |
| 72 | + if not ids: |
| 73 | + return [] |
| 74 | + return [cls.from_dict(_load_fixture(f'{_FOLDERS[cls]}/{_}.json')) for _ in ids] # type: ignore[attr-defined] |
| 75 | + |
| 76 | + |
| 77 | +def _cluster_policy(policy_id: str): |
| 78 | + fixture = _load_fixture(f"assessment/policies/{policy_id}.json") |
| 79 | + definition = json.dumps(fixture["definition"]) |
| 80 | + overrides = json.dumps(fixture["policy_family_definition_overrides"]) |
| 81 | + return Policy(description=definition, policy_family_definition_overrides=overrides) |
| 82 | + |
| 83 | + |
| 84 | +def _pipeline(pipeline_id: str): |
| 85 | + fixture = _load_fixture(f"assessment/pipelines/{pipeline_id}.json") |
| 86 | + return GetPipelineResponse.from_dict(fixture) |
| 87 | + |
| 88 | + |
| 89 | +def _secret_not_found(secret_scope, _): |
| 90 | + msg = f"Secret Scope {secret_scope} does not exist!" |
| 91 | + raise NotFound(msg) |
| 92 | + |
| 93 | + |
| 94 | +def workspace_client_mock( |
| 95 | + cluster_ids: list[str] | None = None, |
| 96 | + pipeline_ids: list[str] | None = None, |
| 97 | + job_ids: list[str] | None = None, |
| 98 | + jobruns_ids: list[str] | None = None, |
| 99 | + policy_ids: list[str] | None = None, |
| 100 | + warehouse_config="single-config.json", |
| 101 | + secret_exists=True, |
| 102 | +): |
| 103 | + ws = create_autospec(WorkspaceClient) |
| 104 | + ws.clusters.list.return_value = _id_list(ClusterDetails, cluster_ids) |
| 105 | + ws.cluster_policies.list.return_value = _id_list(Policy, policy_ids) |
| 106 | + ws.cluster_policies.get = _cluster_policy |
| 107 | + ws.pipelines.list_pipelines.return_value = _id_list(PipelineStateInfo, pipeline_ids) |
| 108 | + ws.pipelines.get = _pipeline |
| 109 | + ws.jobs.list.return_value = _id_list(BaseJob, job_ids) |
| 110 | + ws.jobs.list_runs.return_value = _id_list(BaseRun, jobruns_ids) |
| 111 | + ws.warehouses.get_workspace_warehouse_config().data_access_config = _load_list( |
| 112 | + EndpointConfPair, f"assessment/warehouses/{warehouse_config}" |
| 113 | + ) |
| 114 | + ws.workspace.export = _workspace_export |
| 115 | + if secret_exists: |
| 116 | + ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value="SGVsbG8sIFdvcmxkIQ==") |
| 117 | + else: |
| 118 | + ws.secrets.get_secret = _secret_not_found |
| 119 | + return ws |
| 120 | + |
| 121 | + |
| 122 | +def table_mapping_mock(tables: list[str] | None = None): |
| 123 | + table_mapping = create_autospec(TableMapping) |
| 124 | + table_mapping.get_tables_to_migrate.return_value = _id_list(TableToMigrate, tables) |
| 125 | + return table_mapping |
0 commit comments