Rename TablesMigrate -> TablesMigrator and upgraded -> migrated for clarity and consistency (#1181)

- rename ViewsMigrator to ViewsSequencer for clarity
- rename TableMigrate to TableMigrator
- Rename 'upgrade' methods to 'migrate' for consistency

Author: ericvergnaud

src/databricks/labs/ucx/cli.py

@@ -24,7 +24,7 @@
 from databricks.labs.ucx.hive_metastore import ExternalLocations, TablesCrawler
 from databricks.labs.ucx.hive_metastore.catalog_schema import CatalogSchema
 from databricks.labs.ucx.hive_metastore.mapping import TableMapping
-from databricks.labs.ucx.hive_metastore.table_migrate import TablesMigrate
+from databricks.labs.ucx.hive_metastore.table_migrate import TablesMigrator
 from databricks.labs.ucx.hive_metastore.table_move import TableMove
 from databricks.labs.ucx.install import WorkspaceInstallation
 from databricks.labs.ucx.installer.workflows import WorkflowsInstallation
@@ -205,7 +205,7 @@ def revert_migrated_tables(
         question = "You haven't specified a schema or a table. All migrated tables will be reverted. Continue?"
         if not prompts.confirm(question, max_attempts=2):
             return
-    tables_migrate = TablesMigrate.for_cli(w)
+    tables_migrate = TablesMigrator.for_cli(w)
     revert = tables_migrate.print_revert_report(delete_managed=delete_managed)
     if revert and prompts.confirm("Would you like to continue?", max_attempts=2):
         tables_migrate.revert_migrated_tables(schema, table, delete_managed=delete_managed)

src/databricks/labs/ucx/hive_metastore/mapping.py

@@ -133,7 +133,7 @@ def get_tables_to_migrate(self, tables_crawler: TablesCrawler) -> Collection[Tab
                 logger.info(f"Table {rule.as_hms_table_key} is in a database that was marked to be skipped")
                 continue
             if crawled_tables_keys[rule.as_hms_table_key].is_databricks_dataset:
-                logger.info(f"Table {rule.as_hms_table_key} is a db demo dataset and will not be upgraded")
+                logger.info(f"Table {rule.as_hms_table_key} is a db demo dataset and will not be migrated")
                 continue
             tasks.append(
                 partial(self._get_table_in_scope_task, TableToMigrate(crawled_tables_keys[rule.as_hms_table_key], rule))
@@ -177,8 +177,8 @@ def _get_table_in_scope_task(self, table_to_migrate: TableToMigrate) -> TableToM
                 logger.info(f"{table.key} is set as upgraded to {value['value']}")
                 if self.exists_in_uc(table, value["value"]):
                     logger.info(
-                        f"The table {table.key} was previously upgraded to {value['value']}. "
-                        f"To revert the table and allow it to be upgraded again use the CLI command:"
+                        f"The table {table.key} was previously migrated to {value['value']}. "
+                        f"To revert the table and allow it to be migrated again use the CLI command:"
                         f"databricks labs ucx revert --schema {table.database} --table {table.name}"
                     )
                     return None

src/databricks/labs/ucx/hive_metastore/table_migrate.py

@@ -42,7 +42,7 @@ def destination(self):
         return f"{self.dst_catalog}.{self.dst_schema}.{self.dst_table}".lower()
 
 
-class TablesMigrate:
+class TablesMigrator:
     def __init__(
         self,
         table_crawler: TablesCrawler,
@@ -99,8 +99,8 @@ def migrate_tables(self, *, what: What | None = None, acl_strategy: list[AclMigr
         Threads.strict("migrate tables", tasks)
 
     def _migrate_table(self, src_table: Table, rule: Rule, grants: list[Grant] | None = None):
-        if self._table_already_upgraded(rule.as_uc_table_key):
-            logger.info(f"Table {src_table.key} already upgraded to {rule.as_uc_table_key}")
+        if self._table_already_migrated(rule.as_uc_table_key):
+            logger.info(f"Table {src_table.key} already migrated to {rule.as_uc_table_key}")
             return True
         if src_table.what == What.DBFS_ROOT_DELTA:
             return self._migrate_dbfs_root_table(src_table, rule, grants)
@@ -155,13 +155,13 @@ def _migrate_acl(self, src: Table, rule: Rule, grants: list[Grant] | None):
             self._backend.execute(acl_migrate_sql)
         return True
 
-    def _table_already_upgraded(self, target) -> bool:
+    def _table_already_migrated(self, target) -> bool:
         return target in self._seen_tables
 
     def _get_tables_to_revert(self, schema: str | None = None, table: str | None = None) -> list[Table]:
         schema = schema.lower() if schema else None
         table = table.lower() if table else None
-        upgraded_tables = []
+        migrated_tables = []
         if table and not schema:
             logger.error("Cannot accept 'Table' parameter without 'Schema' parameter")
 
@@ -171,24 +171,24 @@ def _get_tables_to_revert(self, schema: str | None = None, table: str | None = N
             if table and cur_table.name != table:
                 continue
             if cur_table.key in self._seen_tables.values():
-                upgraded_tables.append(cur_table)
-        return upgraded_tables
+                migrated_tables.append(cur_table)
+        return migrated_tables
 
     def revert_migrated_tables(
         self, schema: str | None = None, table: str | None = None, *, delete_managed: bool = False
     ):
         self._init_seen_tables()
-        upgraded_tables = self._get_tables_to_revert(schema=schema, table=table)
+        migrated_tables = self._get_tables_to_revert(schema=schema, table=table)
         # reverses the _seen_tables dictionary to key by the source table
         reverse_seen = {v: k for (k, v) in self._seen_tables.items()}
         tasks = []
-        for upgraded_table in upgraded_tables:
-            if upgraded_table.kind == "VIEW" or upgraded_table.object_type == "EXTERNAL" or delete_managed:
-                tasks.append(partial(self._revert_migrated_table, upgraded_table, reverse_seen[upgraded_table.key]))
+        for migrated_table in migrated_tables:
+            if migrated_table.kind == "VIEW" or migrated_table.object_type == "EXTERNAL" or delete_managed:
+                tasks.append(partial(self._revert_migrated_table, migrated_table, reverse_seen[migrated_table.key]))
                 continue
             logger.info(
-                f"Skipping {upgraded_table.object_type} Table {upgraded_table.database}.{upgraded_table.name} "
-                f"upgraded_to {upgraded_table.upgraded_to}"
+                f"Skipping {migrated_table.object_type} Table {migrated_table.database}.{migrated_table.name} "
+                f"upgraded_to {migrated_table.upgraded_to}"
             )
         Threads.strict("revert migrated tables", tasks)
 
@@ -201,10 +201,10 @@ def _revert_migrated_table(self, table: Table, target_table_key: str):
 
     def _get_revert_count(self, schema: str | None = None, table: str | None = None) -> list[MigrationCount]:
         self._init_seen_tables()
-        upgraded_tables = self._get_tables_to_revert(schema=schema, table=table)
+        migrated_tables = self._get_tables_to_revert(schema=schema, table=table)
 
         table_by_database = defaultdict(list)
-        for cur_table in upgraded_tables:
+        for cur_table in migrated_tables:
             table_by_database[cur_table.database].append(cur_table)
 
         migration_list = []
@@ -217,8 +217,8 @@ def _get_revert_count(self, schema: str | None = None, table: str | None = None)
             migration_list.append(MigrationCount(database=cur_database, what_count=what_count))
         return migration_list
 
-    def is_upgraded(self, schema: str, table: str) -> bool:
-        return self._migration_status_refresher.is_upgraded(schema, table)
+    def is_migrated(self, schema: str, table: str) -> bool:
+        return self._migration_status_refresher.is_migrated(schema, table)
 
     def print_revert_report(self, *, delete_managed: bool) -> bool | None:
         migrated_count = self._get_revert_count()
@@ -280,7 +280,7 @@ class MigrationIndex:
     def __init__(self, tables: list[MigrationStatus]):
         self._index = {(ms.src_schema, ms.src_table): ms for ms in tables}
 
-    def is_upgraded(self, schema: str, table: str) -> bool:
+    def is_migrated(self, schema: str, table: str) -> bool:
         """Check if a table is migrated."""
         return self.get(schema, table) is not None
 
@@ -318,13 +318,13 @@ def get_seen_tables(self) -> dict[str, str]:
                 seen_tables[table.full_name.lower()] = table.properties["upgraded_from"].lower()
         return seen_tables
 
-    def is_upgraded(self, schema: str, table: str) -> bool:
+    def is_migrated(self, schema: str, table: str) -> bool:
         result = self._backend.fetch(f"SHOW TBLPROPERTIES {escape_sql_identifier(schema + '.' + table)}")
         for value in result:
             if value["key"] == "upgraded_to":
-                logger.info(f"{schema}.{table} is set as upgraded")
+                logger.info(f"{schema}.{table} is set as migrated")
                 return True
-        logger.info(f"{schema}.{table} is set as not upgraded")
+        logger.info(f"{schema}.{table} is set as not migrated")
         return False
 
     def _crawl(self) -> Iterable[MigrationStatus]:
@@ -339,7 +339,7 @@ def _crawl(self) -> Iterable[MigrationStatus]:
                 src_table=src_table,
                 update_ts=str(timestamp),
             )
-            if table.key in reverse_seen and self.is_upgraded(src_schema, src_table):
+            if table.key in reverse_seen and self.is_migrated(src_schema, src_table):
                 target_table = reverse_seen[table.key]
                 if len(target_table.split(".")) == 3:
                     table_migration_status.dst_catalog = target_table.split(".")[0]

src/databricks/labs/ucx/hive_metastore/tables.py

@@ -41,6 +41,7 @@ class Table:
 
     location: str | None = None
     view_text: str | None = None
+    # really means migrated_to
     upgraded_to: str | None = None
 
     storage_properties: str | None = None

src/databricks/labs/ucx/hive_metastore/views_migrator.py renamed to src/databricks/labs/ucx/hive_metastore/views_sequencer.py

@@ -3,7 +3,6 @@
 from sqlglot.expressions import Expression as SqlExpression
 from sqlglot.expressions import Table as SqlTable
 
-from databricks.labs.ucx.hive_metastore import TablesCrawler
 from databricks.labs.ucx.hive_metastore.tables import Table
 
 
@@ -73,10 +72,10 @@ def __hash__(self):
         return hash(self._view)
 
 
-class ViewsMigrator:
+class ViewsSequencer:
 
-    def __init__(self, crawler: TablesCrawler):
-        self._crawler = crawler
+    def __init__(self, tables: list[Table]):
+        self._tables = tables
         self._result_view_list: list[ViewToMigrate] = []
         self._result_tables_set: set[Table] = set()
 
@@ -89,10 +88,9 @@ def sequence(self) -> list[Table]:
         # if none, then it's safe to add that view to the next batch of views
         # the complexity for a given set of views v and a dependency depth d looks like Ov^d
         # this seems enormous but in practice d remains small and v decreases rapidly
-        table_list = self._crawler.snapshot()
         all_tables = {}
         views = set()
-        for table in table_list:
+        for table in self._tables:
             all_tables[table.key] = table
             if table.view_text is None:
                 continue

src/databricks/labs/ucx/queries/assessment/estimates/03_0_data_migration.md

@@ -4,15 +4,15 @@
 Once you have defined your data model in UC and that you've created appropriate Storage Credentials and External Locations, 
 you can then migrate your data to UC 
 
-Assumptions for a single table upgrade estimates:
+Assumptions for a single table migration estimates:
 
 - UC Data model has been defined
 - Storage Credentials are in place
 - External Locations are in place
 - Target Catalogs and schemas has been defined
 - Grants has been defined
 
-Please note that depending on the table type and their location, the upgrade effort will differ.
+Please note that depending on the table type and their location, the migration effort will differ.
 [Full guidance](https://www.databricks.com/blog/migrating-tables-hive-metastore-unity-catalog-metastore)
 
 | object_type | table_format                         | location       | estimated effort | suggestion                                                                               |

src/databricks/labs/ucx/runtime.py

@@ -23,7 +23,7 @@
 from databricks.labs.ucx.hive_metastore.mapping import TableMapping
 from databricks.labs.ucx.hive_metastore.table_migrate import (
     MigrationStatusRefresher,
-    TablesMigrate,
+    TablesMigrator,
 )
 from databricks.labs.ucx.hive_metastore.table_size import TableSizeCrawler
 from databricks.labs.ucx.hive_metastore.tables import AclMigrationWhat, What
@@ -437,7 +437,7 @@ def migrate_external_tables_sync(
     table_mapping = TableMapping(install, ws, sql_backend)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, cfg.inventory_database, table_crawler)
     group_manager = GroupManager(sql_backend, ws, cfg.inventory_database)
-    TablesMigrate(
+    TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     ).migrate_tables(what=What.EXTERNAL_SYNC, acl_strategy=[AclMigrationWhat.LEGACY_TACL])
 
@@ -457,7 +457,7 @@ def migrate_dbfs_root_delta_tables(
     table_mapping = TableMapping(install, ws, sql_backend)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, cfg.inventory_database, table_crawler)
     group_manager = GroupManager(sql_backend, ws, cfg.inventory_database)
-    TablesMigrate(
+    TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     ).migrate_tables(what=What.DBFS_ROOT_DELTA, acl_strategy=[AclMigrationWhat.LEGACY_TACL])
 

src/databricks/labs/ucx/source_code/files.py

@@ -4,7 +4,7 @@
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.service.workspace import Language
 
-from databricks.labs.ucx.hive_metastore.table_migrate import TablesMigrate
+from databricks.labs.ucx.hive_metastore.table_migrate import TablesMigrator
 from databricks.labs.ucx.source_code.languages import Languages
 
 logger = logging.getLogger(__name__)
@@ -19,7 +19,7 @@ def __init__(self, languages: Languages):
 
     @classmethod
     def for_cli(cls, ws: WorkspaceClient):
-        tables_migrate = TablesMigrate.for_cli(ws)
+        tables_migrate = TablesMigrator.for_cli(ws)
         index = tables_migrate.index()
         languages = Languages(index)
         return cls(languages)

tests/integration/hive_metastore/test_migrate.py

@@ -12,7 +12,7 @@
 from databricks.labs.ucx.hive_metastore.mapping import Rule
 from databricks.labs.ucx.hive_metastore.table_migrate import (
     MigrationStatusRefresher,
-    TablesMigrate,
+    TablesMigrator,
 )
 from databricks.labs.ucx.hive_metastore.tables import AclMigrationWhat, Table
 from databricks.labs.ucx.workspace_access.groups import GroupManager
@@ -56,7 +56,7 @@ def test_migrate_managed_tables(ws, sql_backend, inventory_schema, make_catalog,
     table_mapping = StaticTableMapping(ws, sql_backend, rules=rules)
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     )
 
@@ -117,7 +117,7 @@ def test_migrate_tables_with_cache_should_not_create_table(
     table_mapping = StaticTableMapping(ws, sql_backend, rules=rules)
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     )
 
@@ -169,7 +169,7 @@ def test_migrate_external_table(  # pylint: disable=too-many-locals
     ]
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler,
         grant_crawler,
         ws,
@@ -227,7 +227,7 @@ def test_migrate_external_table_failed_sync(
     ]
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler,
         grant_crawler,
         ws,
@@ -281,7 +281,7 @@ def test_revert_migrated_table(
     table_mapping = StaticTableMapping(ws, sql_backend, rules=rules)
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     )
     table_migrate.migrate_tables()
@@ -290,11 +290,11 @@ def test_revert_migrated_table(
 
     # Checking that two of the tables were reverted and one was left intact.
     # The first two table belongs to schema 1 and should have not "upgraded_to" property
-    assert not table_migrate.is_upgraded(table_to_revert.schema_name, table_to_revert.name)
+    assert not table_migrate.is_migrated(table_to_revert.schema_name, table_to_revert.name)
     # The second table didn't have the "upgraded_to" property set and should remain that way.
-    assert not table_migrate.is_upgraded(table_not_migrated.schema_name, table_not_migrated.name)
+    assert not table_migrate.is_migrated(table_not_migrated.schema_name, table_not_migrated.name)
     # The third table belongs to schema2 and had the "upgraded_to" property set and should remain that way.
-    assert table_migrate.is_upgraded(table_to_not_revert.schema_name, table_to_not_revert.name)
+    assert table_migrate.is_migrated(table_to_not_revert.schema_name, table_to_not_revert.name)
 
     target_tables_schema1 = list(sql_backend.fetch(f"SHOW TABLES IN {dst_schema1.full_name}"))
     assert len(target_tables_schema1) == 0
@@ -393,7 +393,7 @@ def test_mapping_reverts_table(
     table_mapping = StaticTableMapping(ws, sql_backend, rules=rules)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     )
     table_migrate.migrate_tables()
@@ -482,7 +482,7 @@ def test_migrate_managed_tables_with_acl(
     table_mapping = StaticTableMapping(ws, sql_backend, rules=rules)
     group_manager = GroupManager(sql_backend, ws, inventory_schema)
     migration_status_refresher = MigrationStatusRefresher(ws, sql_backend, inventory_schema, table_crawler)
-    table_migrate = TablesMigrate(
+    table_migrate = TablesMigrator(
         table_crawler, grant_crawler, ws, sql_backend, table_mapping, group_manager, migration_status_refresher
     )