Update existing and add new SQL queries to the assessment step dashboard (#269)

Fixes #268

---------

Co-authored-by: FastLee <[email protected]>

Author: larsgeorge-db

src/databricks/labs/ucx/assessment/queries/all_tables.sql

@@ -0,0 +1,16 @@
+-- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location
+-- widget title=Table Types, col=0, row=3, size_x=6, size_y=6
+SELECT `database`,
+       name,
+       object_type AS type,
+       UPPER(table_format) AS format,
+       IF(object_type IN ("MANAGED", "EXTERNAL"), "TABLE", "VIEW") AS table_view,
+       CASE
+           WHEN STARTSWITH(location, "/dbfs/")
+                AND NOT STARTSWITH(location, "/dbfs/mnt") THEN "DBFS ROOT"
+           WHEN STARTSWITH(location, "/dbfs/mnt") THEN "DBFS MOUNT"
+           ELSE "EXTERNAL"
+       END AS storage,
+       IF(format = "delta", "Yes", "No") AS is_delta,
+       location
+FROM $inventory.tables

src/databricks/labs/ucx/assessment/queries/clusters.sql

@@ -0,0 +1,9 @@
+-- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures
+-- widget title=Clusters, col=0, row=25, size_x=6, size_y=8
+SELECT cluster_id,
+       cluster_name,
+       creator,
+       IF(success = 1, "Compatible", "Incompatible") AS compatible,
+       failures
+FROM $inventory.clusters
+WHERE NOT STARTSWITH(cluster_name, "job-")

src/databricks/labs/ucx/assessment/queries/count_total_databases.sql

@@ -0,0 +1,4 @@
+-- viz type=counter, name=Total Database Count, counter_label=Total Databases, value_column=count_total_databases
+-- widget col=0, row=0, size_x=1, size_y=3
+SELECT COUNT(DISTINCT `database`) AS count_total_databases
+FROM hive_metastore.ucx.tables

src/databricks/labs/ucx/assessment/queries/count_total_tables.sql

@@ -0,0 +1,4 @@
+-- viz type=counter, name=Total Table Count, counter_label=Total Tables, value_column=count_total_tables
+-- widget col=1, row=0, size_x=1, size_y=3
+SELECT count(*) AS count_total_tables
+FROM $inventory.tables

src/databricks/labs/ucx/assessment/queries/database_summary.sql

@@ -0,0 +1,25 @@
+-- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgrade
+-- widget title=Database Summary, col=0, row=9, size_x=6, size_y=8
+SELECT `database`,
+       SUM(is_table) AS tables,
+       SUM(is_view) AS views,
+       SUM(is_dbfs_root) AS dbfs_root,
+       SUM(is_delta) AS delta_tables,
+       CASE
+           WHEN (SUM(is_dbfs_root)/SUM(is_table) > .3) THEN "Asset Replication Required"
+           WHEN (SUM(is_delta)/SUM(is_table) < .7) THEN "Some Non Delta Assets"
+           ELSE "In Place Sync"
+       END AS upgrade
+FROM
+  (SELECT DATABASE,
+          name,
+          object_type,
+          UPPER(table_format) AS format,
+          LOCATION,
+          IF(object_type IN ("MANAGED", "EXTERNAL"), 1, 0) AS is_table,
+          IF(object_type = "VIEW", 1, 0) AS is_view,
+          IF(STARTSWITH(location, "/dbfs/") AND NOT STARTSWITH(location, "/dbfs/mnt"), 1, 0) AS is_dbfs_root,
+          IF(UPPER(format) = "DELTA", 1, 0) AS is_delta
+   FROM $inventory.tables)
+GROUP BY `database`
+ORDER BY `database`

src/databricks/labs/ucx/assessment/queries/external_locations.sql

@@ -0,0 +1,4 @@
+-- viz type=table, name=External Locations, columns=location
+-- widget title=External Locations, col=0, row=17, size_x=3, size_y=8
+SELECT location
+FROM hive_metastore.ucx.external_locations

src/databricks/labs/ucx/assessment/queries/jobs.sql

@@ -0,0 +1,9 @@
+-- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures
+-- widget title=Jobs, col=0, row=33, size_x=6, size_y=8
+SELECT job_id,
+       job_name,
+       creator,
+       IF(success=1, "Compatible", "Incompatible") AS compatible,
+       failures
+FROM $inventory.jobs
+WHERE job_name not like '[UCX]%'

src/databricks/labs/ucx/assessment/queries/mount_points.sql

@@ -0,0 +1,5 @@
+-- viz type=table, name=Mount Points, columns=name,source
+-- widget title=Mount Points, col=3, row=17, size_x=3, size_y=8
+SELECT name,
+       source
+FROM hive_metastore.ucx.mounts