Switch TABLE metadata from SHOW TABLES to SHOW AS RESOURCE TABLES

Author: littleK0i

snowddl/app/base.py

@@ -277,6 +277,12 @@ def init_arguments_parser(self):
             "--destroy-without-prefix", help="Allow {destroy} action without --env-prefix", default=False, action="store_true"
         )
 
+        # Legacy compatibility flags
+        parser.add_argument(
+            "--legacy-column-expression",
+            help="Enable handling for undocumented table column expression parameter",
+        )
+
         # Subparsers
         subparsers = parser.add_subparsers(dest="action")
         subparsers.required = True
@@ -537,6 +543,9 @@ def init_settings(self):
         if self.args.get("max_workers"):
             settings.max_workers = int(self.args.get("max_workers"))
 
+        if self.args.get("legacy_column_expression"):
+            settings.legacy_column_expression = True
+
         return settings
 
     def get_engine(self):

snowddl/resolver/table.py

@@ -1,5 +1,5 @@
 from itertools import islice
-from re import compile
+from json import loads as json_loads
 
 from snowddl.blueprint import (
     Ident,
@@ -12,9 +12,6 @@
 )
 from snowddl.resolver.abc_schema_object_resolver import AbstractSchemaObjectResolver, ResolveResult, ObjectType
 
-cluster_by_syntax_re = compile(r"^(\w+)?\((.*)\)$")
-collate_type_syntax_re = compile(r"^(.*) COLLATE \'(.*)\'$")
-
 
 class TableResolver(AbstractSchemaObjectResolver):
     def get_object_type(self) -> ObjectType:
@@ -24,22 +21,18 @@ def get_existing_objects_in_schema(self, schema: dict):
         existing_objects = {}
 
         cur = self.engine.execute_meta(
-            "SHOW TABLES IN SCHEMA {database:i}.{schema:i}",
+            "SHOW AS RESOURCE TABLES IN SCHEMA {database:i}.{schema:i}",
             {
                 "database": schema["database"],
                 "schema": schema["schema"],
             },
         )
 
         for r in cur:
+            r = json_loads(r["As Resource"])
+
             # Skip other table types
-            if (
-                r.get("is_external") == "Y"
-                or r.get("is_event") == "Y"
-                or r.get("is_hybrid") == "Y"
-                or r.get("is_iceberg") == "Y"
-                or r.get("is_dynamic") == "Y"
-            ):
+            if r["table_type"] != "NORMAL":
                 continue
 
             full_name = f"{r['database_name']}.{r['schema_name']}.{r['name']}"
@@ -49,11 +42,13 @@ def get_existing_objects_in_schema(self, schema: dict):
                 "name": r["name"],
                 "owner": r["owner"],
                 "is_transient": r["kind"] == "TRANSIENT",
-                "retention_time": int(r["retention_time"]),
-                "cluster_by": r["cluster_by"] if r["cluster_by"] else None,
-                "change_tracking": bool(r["change_tracking"] == "ON"),
-                "search_optimization": bool(r.get("search_optimization") == "ON"),
-                "comment": r["comment"] if r["comment"] else None,
+                "retention_time": r["data_retention_time_in_days"],
+                "cluster_by": r["cluster_by"],
+                "change_tracking": r["change_tracking"],
+                "search_optimization": r["search_optimization"],
+                "comment": r["comment"],
+                "columns": r["columns"],
+                "constraints": r["constraints"],
             }
 
         return existing_objects
@@ -77,7 +72,7 @@ def compare_object(self, bp: TableBlueprint, row: dict):
         replace_notices = []
 
         bp_cols = {str(c.name): c for c in bp.columns}
-        snow_cols = self._get_existing_columns(bp)
+        snow_cols = self._get_existing_columns(bp, row)
 
         dropping_col_names = []
         remaining_col_names = list(snow_cols.keys())
@@ -381,36 +376,35 @@ def drop_object(self, row: dict):
 
         return ResolveResult.DROP
 
-    def _get_existing_columns(self, bp: TableBlueprint):
+    def _get_existing_columns(self, bp: TableBlueprint, row: dict):
         existing_columns = {}
 
-        cur = self.engine.execute_meta(
-            "DESC TABLE {full_name:i}",
-            {
-                "full_name": bp.full_name,
-            },
-        )
-
-        for r in cur:
-            m = collate_type_syntax_re.match(r["type"])
+        for c in row["columns"]:
+            existing_columns[c["name"]] = TableColumn(
+                name=Ident(c["name"]),
+                type=DataType(c["datatype"]),
+                not_null=c["nullable"] is False,
+                default=c["default"],
+                collate=c["collate"],
+                comment=c["comment"],
+            )
 
-            if m:
-                dtype = m.group(1)
-                collate = m.group(2)
-            else:
-                dtype = r["type"]
-                collate = None
-
-            existing_columns[r["name"]] = TableColumn(
-                name=Ident(r["name"]),
-                type=DataType(dtype),
-                not_null=bool(r["null?"] == "N"),
-                default=r["default"] if r["default"] else None,
-                expression=r["expression"] if r["expression"] else None,
-                collate=collate,
-                comment=r["comment"] if r["comment"] else None,
+        # Expression is not available in SHOW AS RESOURCE TABLES output
+        # DESC TABLE must be used explicitly in order to read expression value
+        if self.engine.settings.legacy_column_expression:
+            cur = self.engine.execute_meta(
+                "DESC TABLE {full_name:i}",
+                {
+                    "full_name": bp.full_name,
+                },
             )
 
+            for r in cur:
+                if r["expression"] is None:
+                    continue
+
+                existing_columns[r["name"]].expression = r["expression"]
+
         return existing_columns
 
     def _build_create_table(self, bp: TableBlueprint, snow_cols=None):
@@ -574,10 +568,10 @@ def _build_create_table(self, bp: TableBlueprint, snow_cols=None):
         return query
 
     def _compare_cluster_by(self, bp: TableBlueprint, row: dict):
-        bp_cluster_by = ", ".join(bp.cluster_by) if bp.cluster_by else None
-        snow_cluster_by = cluster_by_syntax_re.sub(r"\2", row["cluster_by"]) if row["cluster_by"] else None
+        bp_normalized_cluster_by = [c.upper() for c in bp.cluster_by] if bp.cluster_by else None
+        snow_normalized_cluster_by = [c.upper() for c in row["cluster_by"]] if row["cluster_by"] else None
 
-        return bp_cluster_by == snow_cluster_by
+        return bp_normalized_cluster_by == snow_normalized_cluster_by
 
     def _create_search_optimization(self, bp: TableBlueprint, condition=True):
         # Legacy search optimization on an entire table

snowddl/settings.py

@@ -31,6 +31,9 @@ class SnowDDLSettings(BaseModelWithConfig):
     ignore_ownership: bool = False
     max_workers: int = 32
 
+    # Legacy compatibility flags
+    legacy_column_expression: bool = False
+
     # Options specific for snowddl-convert
     convert_function_body_to_file: bool = False
     convert_view_text_to_file: bool = False