Skip to content

Remove references to UCX #56

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 30, 2024
Merged

Remove references to UCX #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c92c4b
Remove references to UCX
JCZuurmond Sep 30, 2024
6ac4940
Align name logic
JCZuurmond Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions src/databricks/labs/pytester/fixtures/catalog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,7 @@ def create( # pylint: disable=too-many-locals,too-many-arguments,too-many-state
schema = make_schema(catalog_name=catalog_name)
catalog_name = schema.catalog_name
schema_name = schema.name
if name is None:
name = f"ucx_T{make_random(4)}".lower()
name = name or f"dummy_t{make_random(4).lower()}"
table_type: TableType | None = None
data_source_format = None
storage_location = None
Expand All @@ -140,7 +139,7 @@ def create( # pylint: disable=too-many-locals,too-many-arguments,too-many-state
table_type = TableType.EXTERNAL # pylint: disable=redefined-variable-type
data_source_format = DataSourceFormat.JSON
# DBFS locations are not purged; no suffix necessary.
storage_location = f"dbfs:/tmp/ucx_test_{make_random(4)}"
storage_location = f"dbfs:/tmp/{name}"
if columns is None:
select = "*"
else:
Expand Down Expand Up @@ -264,8 +263,7 @@ def test_catalog_fixture(make_catalog, make_schema, make_table):
"""

def create(*, catalog_name: str = "hive_metastore", name: str | None = None) -> SchemaInfo:
if name is None:
name = f"dummy_S{make_random(4)}".lower()
name = name or f"dummy_s{make_random(4)}".lower()
full_name = f"{catalog_name}.{name}".lower()
sql_backend.execute(f"CREATE SCHEMA {full_name} WITH DBPROPERTIES (RemoveAfter={watchdog_remove_after})")
schema_info = SchemaInfo(catalog_name=catalog_name, name=name, full_name=full_name)
Expand Down Expand Up @@ -301,7 +299,7 @@ def test_catalog_fixture(make_catalog, make_schema, make_table):
"""

def create(*, name: str | None = None) -> CatalogInfo:
name = name or f"dummy_C{make_random(4)}".lower()
name = name or f"dummy_c{make_random(4)}".lower()
catalog_info = ws.catalogs.create(name=name, properties={"RemoveAfter": watchdog_remove_after})
if isinstance(catalog_info, Mock):
catalog_info.name = name
Expand Down Expand Up @@ -353,8 +351,7 @@ def create(
catalog_name = schema.catalog_name
schema_name = schema.name

if name is None:
name = f"ucx_t{make_random(4).lower()}"
name = name or f"dummy_f{make_random(4)}".lower()

# Note: the Watchdog does not explicitly scan for functions; they are purged along with their parent schema.
# As such the function can't be marked (and doesn't need to be if the schema as marked) for purge protection.
Expand Down
2 changes: 1 addition & 1 deletion src/databricks/labs/pytester/fixtures/redash.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def create(sql_query: str | None = None, **kwargs) -> LegacyQuery:
query_name = f"dummy_query_Q{make_random(4)}"
query = ws.queries_legacy.create(
name=query_name,
description="TEST QUERY FOR UCX",
description="Test query",
query=sql_query,
tags=tags,
)
Expand Down
56 changes: 28 additions & 28 deletions tests/unit/fixtures/test_catalog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@ def test_make_table_no_args():
assert table_info == TableInfo(
catalog_name='hive_metastore',
schema_name='dummy_srandom',
name='ucx_trandom',
name='dummy_trandom',
table_type=TableType.MANAGED,
data_source_format=DataSourceFormat.DELTA,
full_name='hive_metastore.dummy_srandom.ucx_trandom',
storage_location='dbfs:/user/hive/warehouse/dummy_srandom/ucx_trandom',
full_name='hive_metastore.dummy_srandom.dummy_trandom',
storage_location='dbfs:/user/hive/warehouse/dummy_srandom/dummy_trandom',
properties={'RemoveAfter': '2024091313'},
)

assert ctx['sql_backend'].queries == [
'CREATE SCHEMA hive_metastore.dummy_srandom WITH DBPROPERTIES (RemoveAfter=2024091313)',
"CREATE TABLE hive_metastore.dummy_srandom.ucx_trandom (id INT, value STRING) TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.ucx_trandom',
"CREATE TABLE hive_metastore.dummy_srandom.dummy_trandom (id INT, value STRING) TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.dummy_trandom',
'DROP SCHEMA IF EXISTS hive_metastore.dummy_srandom CASCADE',
]

Expand All @@ -34,18 +34,18 @@ def test_make_view():
assert table_info == TableInfo(
catalog_name='hive_metastore',
schema_name='dummy_srandom',
name='ucx_trandom',
name='dummy_trandom',
table_type=TableType.VIEW,
full_name='hive_metastore.dummy_srandom.ucx_trandom',
full_name='hive_metastore.dummy_srandom.dummy_trandom',
properties={'RemoveAfter': '2024091313'},
view_definition='SELECT 1',
)

assert ctx['sql_backend'].queries == [
'CREATE SCHEMA hive_metastore.dummy_srandom WITH DBPROPERTIES (RemoveAfter=2024091313)',
"CREATE VIEW hive_metastore.dummy_srandom.ucx_trandom AS SELECT 1",
"ALTER VIEW hive_metastore.dummy_srandom.ucx_trandom SET TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.ucx_trandom',
"CREATE VIEW hive_metastore.dummy_srandom.dummy_trandom AS SELECT 1",
"ALTER VIEW hive_metastore.dummy_srandom.dummy_trandom SET TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.dummy_trandom',
'DROP SCHEMA IF EXISTS hive_metastore.dummy_srandom CASCADE',
]

Expand All @@ -56,27 +56,27 @@ def test_make_external_table():
assert table_info == TableInfo(
catalog_name='hive_metastore',
schema_name='dummy_srandom',
name='ucx_trandom',
name='dummy_trandom',
table_type=TableType.EXTERNAL,
data_source_format=DataSourceFormat.JSON,
full_name='hive_metastore.dummy_srandom.ucx_trandom',
storage_location='dbfs:/tmp/ucx_test_RANDOM',
full_name='hive_metastore.dummy_srandom.dummy_trandom',
storage_location='dbfs:/tmp/dummy_trandom',
properties={'RemoveAfter': '2024091313'},
)

ctx['log_workspace_link'].assert_called_with(
'hive_metastore.dummy_srandom.ucx_trandom schema',
'explore/data/hive_metastore/dummy_srandom/ucx_trandom',
'hive_metastore.dummy_srandom.dummy_trandom schema',
'explore/data/hive_metastore/dummy_srandom/dummy_trandom',
)

assert ctx['sql_backend'].queries == [
'CREATE SCHEMA hive_metastore.dummy_srandom WITH DBPROPERTIES (RemoveAfter=2024091313)',
'CREATE TABLE hive_metastore.dummy_srandom.ucx_trandom USING json location '
"'dbfs:/tmp/ucx_test_RANDOM' as SELECT CAST(calories_burnt AS INT) AS `id`, "
'CREATE TABLE hive_metastore.dummy_srandom.dummy_trandom USING json location '
"'dbfs:/tmp/dummy_trandom' as SELECT CAST(calories_burnt AS INT) AS `id`, "
'CAST(device_id AS STRING) AS `value` FROM '
'JSON.`dbfs:/databricks-datasets/iot-stream/data-device`',
"ALTER TABLE hive_metastore.dummy_srandom.ucx_trandom SET TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.ucx_trandom',
"ALTER TABLE hive_metastore.dummy_srandom.dummy_trandom SET TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.dummy_trandom',
'DROP SCHEMA IF EXISTS hive_metastore.dummy_srandom CASCADE',
]

Expand All @@ -87,18 +87,18 @@ def test_make_table_custom_schema():
assert table_info == TableInfo(
catalog_name='hive_metastore',
schema_name='dummy_srandom',
name='ucx_trandom',
name='dummy_trandom',
table_type=TableType.MANAGED,
data_source_format=DataSourceFormat.DELTA,
full_name='hive_metastore.dummy_srandom.ucx_trandom',
storage_location='dbfs:/user/hive/warehouse/dummy_srandom/ucx_trandom',
full_name='hive_metastore.dummy_srandom.dummy_trandom',
storage_location='dbfs:/user/hive/warehouse/dummy_srandom/dummy_trandom',
properties={'RemoveAfter': '2024091313'},
)

assert ctx['sql_backend'].queries == [
'CREATE SCHEMA hive_metastore.dummy_srandom WITH DBPROPERTIES (RemoveAfter=2024091313)',
"CREATE TABLE hive_metastore.dummy_srandom.ucx_trandom (`a` INT, `b` STRING) TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.ucx_trandom',
"CREATE TABLE hive_metastore.dummy_srandom.dummy_trandom (`a` INT, `b` STRING) TBLPROPERTIES ( 'RemoveAfter' = '2024091313' )",
'DROP TABLE IF EXISTS hive_metastore.dummy_srandom.dummy_trandom',
'DROP SCHEMA IF EXISTS hive_metastore.dummy_srandom CASCADE',
]

Expand All @@ -120,15 +120,15 @@ def test_make_udf():
assert fn_info == FunctionInfo(
catalog_name='hive_metastore',
schema_name='dummy_srandom',
name='ucx_trandom',
full_name='hive_metastore.dummy_srandom.ucx_trandom',
name='dummy_frandom',
full_name='hive_metastore.dummy_srandom.dummy_frandom',
)

assert ctx['sql_backend'].queries == [
'CREATE SCHEMA hive_metastore.dummy_srandom WITH DBPROPERTIES (RemoveAfter=2024091313)',
'CREATE FUNCTION hive_metastore.dummy_srandom.ucx_trandom(x INT) RETURNS '
'CREATE FUNCTION hive_metastore.dummy_srandom.dummy_frandom(x INT) RETURNS '
'FLOAT CONTAINS SQL DETERMINISTIC RETURN 0;',
'DROP FUNCTION IF EXISTS hive_metastore.dummy_srandom.ucx_trandom',
'DROP FUNCTION IF EXISTS hive_metastore.dummy_srandom.dummy_frandom',
'DROP SCHEMA IF EXISTS hive_metastore.dummy_srandom CASCADE',
]

Expand Down
Loading