Merge pull request #991 from Altinity/backports/25.6/82301_awsglue_filter_columns_response

Antalya 25.6: Backport of ClickHouse#82301 - Filter columns in response from AWSGlue

Author: Enmk

src/Databases/DataLake/GlueCatalog.cpp

@@ -288,6 +288,13 @@ void GlueCatalog::getTableMetadata(
             {
                 const auto column_params = column.GetParameters();
                 bool can_be_nullable = column_params.contains("iceberg.field.optional") && column_params.at("iceberg.field.optional") == "true";
+
+                /// Skip field if it's not "current" (for example Renamed). No idea how someone can utilize "non current fields" but for some reason
+                /// they are returned by Glue API. So if you do "RENAME COLUMN a to new_a" glue will return two fields: a and new_a.
+                /// And a will be marked as "non current" field.
+                if (column_params.contains("iceberg.field.current") && column_params.at("iceberg.field.current") == "false")
+                    continue;
+
                 schema.push_back({column.GetName(), getType(column.GetType(), can_be_nullable)});
             }
             result.setSchema(schema);

tests/integration/test_database_glue/test.py

@@ -345,3 +345,43 @@ def test_hide_sensitive_info(started_cluster):
     )
     assert "SECRET_1" not in node.query(f"SHOW CREATE DATABASE {CATALOG_NAME}")
     assert "SECRET_2" not in node.query(f"SHOW CREATE DATABASE {CATALOG_NAME}")
+
+
+def test_select_after_rename(started_cluster):
+    node = started_cluster.instances["node1"]
+
+    test_ref = f"test_list_tables_{uuid.uuid4()}"
+    table_name = f"{test_ref}_table"
+    root_namespace = f"{test_ref}_namespace"
+
+    namespaces_to_create = [
+        root_namespace,
+        f"{root_namespace}_A",
+        f"{root_namespace}_B",
+        f"{root_namespace}_C",
+    ]
+
+    catalog = load_catalog_impl(started_cluster)
+
+    for namespace in namespaces_to_create:
+        catalog.create_namespace(namespace)
+        assert len(catalog.list_tables(namespace)) == 0
+
+    for namespace in namespaces_to_create:
+        table = create_table(catalog, namespace, table_name)
+
+        num_rows = 10
+        df = generate_arrow_data(num_rows)
+        table.append(df)
+
+        create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME)
+
+        expected = DEFAULT_CREATE_TABLE.format(CATALOG_NAME, namespace, table_name)
+        assert expected == node.query(
+            f"SHOW CREATE TABLE {CATALOG_NAME}.`{namespace}.{table_name}`"
+        )
+
+        with table.update_schema() as update:
+            update.rename_column("bid", "new_bid")
+
+        print(node.query(f"SELECT * FROM {CATALOG_NAME}.`{namespace}.{table_name}`"))