Merge remote-tracking branch 'origin/master' into vector_search_pre_and_post_filtering

Author: rschu1ze

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -167,6 +167,7 @@ CityHash
 Clangd
 ClickBench
 ClickCat
+ClickHaskell
 ClickHouse
 ClickHouse's
 ClickHouseClient

docs/en/interfaces/third-party/client-libraries.md

@@ -83,3 +83,4 @@ ClickHouse Inc does **not** maintain the libraries listed below and hasn't done
  - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
 ### Haskell {#haskell}
  - [hdbc-clickhouse](https://github.com/zaneli/hdbc-clickhouse)
+ - [ClickHaskell](https://clickhaskell.dev/)

src/Databases/DatabaseReplicated.cpp

@@ -24,6 +24,7 @@
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/InterpreterCreateQuery.h>
 #include <Interpreters/InterpreterSetQuery.h>
+#include <Interpreters/NormalizeSelectWithUnionQueryVisitor.h>
 #include <Interpreters/ReplicatedDatabaseQueryStatusSource.h>
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/executeDDLQueryOnCluster.h>
@@ -66,6 +67,7 @@ namespace Setting
     extern const SettingsDistributedDDLOutputMode distributed_ddl_output_mode;
     extern const SettingsInt64 distributed_ddl_task_timeout;
     extern const SettingsBool throw_on_unsupported_query_inside_transaction;
+    extern const SettingsSetOperationMode union_default_mode;
 }
 
 namespace ServerSetting
@@ -1463,8 +1465,11 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
                 }
 
                 auto query_ast = parseQueryFromMetadataInZooKeeper(table_name, create_query_string);
-
                 auto create_query_context = make_query_context();
+
+                NormalizeSelectWithUnionQueryVisitor::Data data{create_query_context->getSettingsRef()[Setting::union_default_mode]};
+                NormalizeSelectWithUnionQueryVisitor{data}.visit(query_ast);
+
                 /// Check larger comment in DatabaseOnDisk::createTableFromAST
                 /// TL;DR applySettingsFromQuery will move the settings from engine to query level
                 /// making it possible to overcome a backward incompatible change.

src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp

@@ -98,13 +98,19 @@ void OptimizeIfWithConstantConditionVisitorData::visit(ASTFunction & function_no
     if (tryExtractConstValueFromCondition(condition_expr, condition))
     {
         ASTPtr replace_ast = condition ? then_expr : else_expr;
+        bool replacement_supports_alias = dynamic_cast<ASTWithAlias *>(replace_ast.get());
+        String if_alias = ast->tryGetAlias();
+        /// We cannot set the resulting alias if the replace ast does not support it (e.g. ASTAsterisk), so it's better to do nothing
+        if (!if_alias.empty() && !replacement_supports_alias)
+            return;
+
         ASTPtr child_copy = ast;
         String replace_alias = replace_ast->tryGetAlias();
-        String if_alias = ast->tryGetAlias();
 
         if (replace_alias.empty())
         {
-            replace_ast->setAlias(if_alias);
+            if (!if_alias.empty())
+                replace_ast->setAlias(if_alias);
             ast = replace_ast;
         }
         else
@@ -113,12 +119,14 @@ void OptimizeIfWithConstantConditionVisitorData::visit(ASTFunction & function_no
             /// But IAST has only method for deep copy of subtree.
             /// This can be a reason of performance degradation in case of deep queries.
             ASTPtr replace_ast_deep_copy = replace_ast->clone();
-            replace_ast_deep_copy->setAlias(if_alias);
+            if (!if_alias.empty())
+                replace_ast_deep_copy->setAlias(if_alias);
             ast = replace_ast_deep_copy;
         }
 
         if (!if_alias.empty())
         {
+            ast->setAlias(if_alias);
             auto alias_it = aliases.find(if_alias);
             if (alias_it != aliases.end() && alias_it->second.get() == child_copy.get())
                 alias_it->second = ast;

src/Interpreters/TreeRewriter.cpp

@@ -366,9 +366,12 @@ void renameDuplicatedColumns(const ASTSelectQuery * select_query)
     for (auto & expr : elements)
     {
         auto name = expr->getAliasOrColumnName();
-
         if (!assigned_column_names.insert(name).second)
         {
+            /// We can't rename with aliases if it doesn't support alias (e.g. asterisk)
+            if (!dynamic_cast<ASTWithAlias *>(expr.get()))
+                continue;
+
             size_t i = 1;
             while (all_column_names.end() != all_column_names.find(name + "_" + toString(i)))
                 ++i;

tests/integration/test_matview_union_replicated/__init__.py

@@ -0,0 +1,86 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import assert_eq_with_retry
+
+cluster = ClickHouseCluster(__file__)
+
+# Create two nodes - one main node and one replica
+node1 = cluster.add_instance("node1", with_zookeeper=True)
+node2 = cluster.add_instance("node2", with_zookeeper=True)
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+def test_matview_union_replicated(started_cluster):
+
+    # Create replicated database, source and target tables and matview
+    node1.query("DROP DATABASE IF EXISTS union_test_replicated SYNC")
+    node1.query("CREATE DATABASE union_test_replicated ENGINE=Replicated('/test/union_replica' , 'shard1', 'replica' || '1');")
+
+    node1.query("""
+    CREATE TABLE union_test_replicated.source_1
+    (
+        timestamp DateTime,
+        value Float64
+    )
+    ENGINE = ReplicatedMergeTree
+    ORDER BY timestamp
+    """)
+
+    node1.query("""
+    CREATE TABLE union_test_replicated.source_2
+    (
+        timestamp DateTime,
+        value Float64
+    )
+    ENGINE = ReplicatedMergeTree
+    ORDER BY timestamp
+    """)
+
+    node1.query("""
+    CREATE TABLE union_test_replicated.target
+    (
+        timestamp DateTime,
+        value Float64
+    )
+    ENGINE = ReplicatedMergeTree
+    ORDER BY timestamp
+    """)
+    
+    node1.query("""
+    CREATE MATERIALIZED VIEW union_test_replicated.mv_test TO union_test_replicated.target AS
+    WITH source_data AS
+    (
+        SELECT timestamp, value FROM union_test_replicated.source_1
+        UNION ALL
+        SELECT timestamp, value FROM union_test_replicated.source_2
+    )
+    SELECT timestamp, value FROM source_data
+    """)
+
+    # Verify INSERT works on Node #1
+    node1.query("INSERT INTO union_test_replicated.source_1 VALUES (now(), 1)")
+
+    # Attach replica on second node
+    node2.query("DROP DATABASE IF EXISTS union_test_replicated SYNC")
+    node2.query("CREATE DATABASE union_test_replicated ENGINE=Replicated('/test/union_replica' , 'shard1', 'replica' || '2');")
+    node2.query("SYSTEM SYNC DATABASE REPLICA union_test_replicated")
+
+    # Verify the table structure on replica
+    assert_eq_with_retry(
+        node2,
+        "DESCRIBE TABLE union_test_replicated.source_1",
+        "timestamp\tDateTime\t\t\t\t\nvalue\tFloat64\t\t\t\t\n"
+    )
+
+    # Run INSERT on replica
+    node2.query("INSERT INTO union_test_replicated.source_1 VALUES (now(), 1)")
+
+    # Clean up
+    node1.query("DROP DATABASE IF EXISTS union_test_replicated SYNC")
+    node2.query("DROP DATABASE IF EXISTS union_test_replicated SYNC")

tests/integration/test_matview_union_replicated/test.py

@@ -3,8 +3,8 @@ SET join_algorithm = 'hash';
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
-CREATE TABLE t1 (`id` Int32, key String, key2 String) ENGINE = MergeTree ORDER BY id;
-CREATE TABLE t2 (`id` Int32, key String, key2 String) ENGINE = MergeTree ORDER BY id;
+CREATE TABLE t1 (`id` Int32, key String, key2 String) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity=8192;
+CREATE TABLE t2 (`id` Int32, key String, key2 String) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity=8192;
 INSERT INTO t1 VALUES (1, '111', '111'),(2, '222', '2'),(2, '222', '222'),(3, '333', '333');
 INSERT INTO t2 VALUES (2, 'AAA', 'AAA'),(2, 'AAA', 'a'),(3, 'BBB', 'BBB'),(4, 'CCC', 'CCC');
 
@@ -92,4 +92,4 @@ INNER JOIN system.processes AS b
 ON (a.query_id = b.query_id) AND (a.query_id = b.query_id)
 WHERE current_database = currentDatabase()
 FORMAT Null
-SETTINGS query_plan_use_new_logical_join_step = true, query_plan_convert_join_to_in = true;
+SETTINGS query_plan_use_new_logical_join_step = true, query_plan_convert_join_to_in = true;

tests/queries/0_stateless/03355_join_to_in_optimization.sql

@@ -0,0 +1 @@
+CREATE TABLE t0 (c0 Int ALIAS if(NULL, 1, *)) ENGINE = Memory; -- { serverError UNKNOWN_IDENTIFIER }