Fixed guess_external_locations failure with AttributeError: as_dict and added an integration test (#259)

Created unit testing
Still seeing weird behavior of the Row class.
Possibly related to Python version.

Fix #256

---------

Co-authored-by: Serge Smertin <[email protected]>

Author: FastLee

src/databricks/labs/ucx/hive_metastore/data_objects.py

@@ -6,6 +6,7 @@
 
 from databricks.labs.ucx.framework.crawlers import CrawlerBase, SqlBackend
 from databricks.labs.ucx.hive_metastore.list_mounts import Mounts
+from databricks.labs.ucx.mixins.sql import Row
 
 
 @dataclass
@@ -20,9 +21,9 @@ def __init__(self, ws: WorkspaceClient, sbe: SqlBackend, schema):
         super().__init__(sbe, "hive_metastore", schema, "external_locations")
         self._ws = ws
 
-    def _external_locations(self, tables, mounts):
+    def _external_locations(self, tables: list[Row], mounts) -> list[ExternalLocation]:
         min_slash = 2
-        external_locations = []
+        external_locations: list[ExternalLocation] = []
         for table in tables:
             location = table.as_dict()["location"]
             if location is not None and len(location) > 0:
@@ -52,7 +53,7 @@ def _external_locations(self, tables, mounts):
         return external_locations
 
     def _external_location_list(self):
-        tables = self._backend.fetch(f"SELECT location FROM {self._schema}.tables WHERE location IS NOT NULL")
+        tables = list(self._backend.fetch(f"SELECT location FROM {self._schema}.tables WHERE location IS NOT NULL"))
         mounts = Mounts(self._backend, self._ws, self._schema).snapshot()
         return self._external_locations(list(tables), list(mounts))
 

tests/integration/hive_metastore/test_external_locations.py

@@ -0,0 +1,29 @@
+import logging
+import os
+
+from databricks.labs.ucx.framework.crawlers import StatementExecutionBackend
+from databricks.labs.ucx.hive_metastore.data_objects import ExternalLocationCrawler
+from databricks.labs.ucx.hive_metastore.list_mounts import Mount
+from databricks.labs.ucx.hive_metastore.tables import Table
+
+logger = logging.getLogger(__name__)
+
+
+def test_table_inventory(ws, make_warehouse, make_schema):
+    warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"]
+
+    logger.info("setting up fixtures")
+    sbe = StatementExecutionBackend(ws, warehouse_id)
+    tables = [
+        Table("hive_metastore", "foo", "bar", "MANAGED", "delta", location="s3://test_location/test1/table1"),
+        Table("hive_metastore", "foo", "bar", "EXTERNAL", "delta", location="s3://test_location/test2/table2"),
+        Table("hive_metastore", "foo", "bar", "EXTERNAL", "delta", location="dbfs:/mnt/foo/test3/table3"),
+    ]
+    schema = make_schema()
+    sbe.save_table(f"{schema}.tables", tables)
+    sbe.save_table(f"{schema}.mounts", [Mount("/mnt/foo", "s3://bar")])
+
+    crawler = ExternalLocationCrawler(ws, sbe, schema.split(".")[1])
+    results = crawler.snapshot()
+    assert len(results) == 2
+    assert results[1].location == "s3://bar/test3/"