Added dashboard widget with table count by storage and format (#852)

Author: dipankarkush-db

docs/assessment.md

@@ -25,15 +25,22 @@ Total number of identified storage locations based on scanning Hive Metastore ta
 # Assessment Widgets
 Assessment widgets query tables in the $inventory database and summarize or detail out findings.
 
-The second row of the report starts with "Readiness" and "Assessment Summary"
-<img width="1235" alt="image" src="https://github.com/databrickslabs/ucx/assets/1122251/c68194c4-4b09-4c8d-b61f-ebf57b7106c7">
+The second row of the report starts with "Readiness", "Assessment Summary", "Table counts by storage" and "Table counts by schema and format"
+
+<img width="1512" alt="image" src="https://github.com/databrickslabs/ucx/assets/106815134/41904d8a-c746-4191-be08-2e9e2090935d">
 
 ## Readiness
 This is a rough summary of the workspace readiness to run Unity Catalog governed workloads. Each line item is the percent of compatible items divided by the total items in the class.
 
 ## Assessment Summary
 This is a summary count, per finding type of all of the findings identified during the assessment workflow. The assessment summary will help identify areas that need focus (e.g. Tables on DBFS or Clusters that need DBR upgrades)
 
+## Table counts by storage
+This is a summary count of Hive Metastore tables, per storage type (DBFS Root, DBFS Mount, Cloud Storage (referred as External)). This also gives a summary count of tables using storage types which are unsupported (such as WASB or ADL in Azure) in Unity Catalog. Count of tables created using Databricks Demo Datasets are also identified here
+
+## Table counts by schema and format
+This is a summary count by Hive Metastore (HMS) table formats (Delta and Non Delta) for each HMS schema    
+
 The third row continues with "Database Summary"
 <img width="1220" alt="image" src="https://github.com/databrickslabs/ucx/assets/1122251/28742e33-d3e3-4eb8-832f-1edd34999fa2">
 

src/databricks/labs/ucx/queries/assessment/main/00_6_count_table_by_storage.sql

@@ -0,0 +1,19 @@
+-- viz type=table, name=Table counts by storage, columns=Storage,count
+-- widget title=Table counts by storage, row=2, col=2, size_x=2, size_y=5
+SELECT storage, COUNT(*) count
+FROM (
+SELECT
+       CASE
+           WHEN STARTSWITH(location, "dbfs:/mnt") THEN "DBFS MOUNT"
+           WHEN STARTSWITH(location, "/dbfs/mnt") THEN "DBFS MOUNT"
+           WHEN STARTSWITH(location, "dbfs:/databricks-datasets") THEN "Databricks Demo Dataset"
+           WHEN STARTSWITH(location, "/dbfs/databricks-datasets") THEN "Databricks Demo Dataset"
+           WHEN STARTSWITH(location, "dbfs:/") THEN "DBFS ROOT"
+           WHEN STARTSWITH(location, "/dbfs/") THEN "DBFS ROOT"
+           WHEN STARTSWITH(location, "wasb") THEN "UNSUPPORTED"
+           WHEN STARTSWITH(location, "adl") THEN "UNSUPPORTED"
+           ELSE "EXTERNAL"
+       END AS storage
+FROM $inventory.tables)
+GROUP BY storage
+ORDER BY storage;

src/databricks/labs/ucx/queries/assessment/main/00_7_count_table_by_schema_format.sql

@@ -0,0 +1,22 @@
+-- viz type=table, name=Table counts by schema and format, columns=schema,format,count
+-- widget title=Table counts by schema and format, row=2, col=4, size_x=2, size_y=5
+SELECT
+  schema,
+  format,
+  COUNT(*) count
+FROM
+  (
+    SELECT
+      `database` AS schema,
+      IF(table_format = 'DELTA', "Delta", "Non Delta") AS format
+    FROM
+      $inventory.tables
+  )
+WHERE
+  format IS NOT NULL
+GROUP BY
+  schema,
+  format
+ORDER BY
+  schema,
+  format;

src/databricks/labs/ucx/queries/assessment/main/01_2_assessment_summary.sql

@@ -1,7 +1,7 @@
 -- viz type=table, name=Assessment Summary, search_by=failure, columns=failure,count
--- widget title=Assessment Summary, row=2, col=2, size_x=4, size_y=11
+-- widget title=Assessment Summary, row=1, col=2, size_x=4, size_y=6
 WITH raw AS (
   SELECT EXPLODE(FROM_JSON(failures, 'array<string>')) AS failure FROM $inventory.objects WHERE failures <> '[]'
 )
 SELECT failure as `item`, COUNT(*) AS count FROM raw GROUP BY failure
-ORDER BY count DESC, failure DESC
+ORDER BY count DESC, failure DESC