Merge pull request ClickHouse#79696 from ianton-ru/macros_for_autodiscovery

Resolve macros for autodiscovery clusters

Author: tuanpach

src/Interpreters/ClusterDiscovery.cpp

@@ -121,10 +121,12 @@ class ClusterDiscovery::Flags
 ClusterDiscovery::ClusterDiscovery(
     const Poco::Util::AbstractConfiguration & config,
     ContextPtr context_,
+    MultiVersion<Macros>::Version macros_,
     const String & config_prefix)
     : context(Context::createCopy(context_))
     , current_node_name(toString(ServerUUID::get()))
     , log(getLogger("ClusterDiscovery"))
+    , macros(macros_)
 {
     LOG_DEBUG(log, "Cluster discovery is enabled");
 
@@ -735,7 +737,8 @@ bool ClusterDiscovery::runMainThread(std::function<void()> up_to_date_callback)
 ClusterPtr ClusterDiscovery::getCluster(const String & cluster_name) const
 {
     std::lock_guard lock(mutex);
-    auto it = cluster_impls.find(cluster_name);
+    auto expanded_cluster_name = macros->expand(cluster_name);
+    auto it = cluster_impls.find(expanded_cluster_name);
     if (it == cluster_impls.end())
         return nullptr;
     return it->second;

src/Interpreters/ClusterDiscovery.h

@@ -28,6 +28,7 @@ class ClusterDiscovery
     ClusterDiscovery(
         const Poco::Util::AbstractConfiguration & config,
         ContextPtr context_,
+        MultiVersion<Macros>::Version macros_,
         const String & config_prefix = "remote_servers");
 
     void start();
@@ -204,6 +205,8 @@ class ClusterDiscovery
     };
 
     std::shared_ptr<std::vector<std::shared_ptr<MulticlusterDiscovery>>> multicluster_discovery_paths;
+
+    MultiVersion<Macros>::Version macros;
 };
 
 }

src/Interpreters/Context.cpp

@@ -4655,7 +4655,7 @@ void Context::setClustersConfig(const ConfigurationPtr & config, bool enable_dis
     std::lock_guard lock(shared->clusters_mutex);
     if (ConfigHelper::getBool(*config, "allow_experimental_cluster_discovery") && enable_discovery && !shared->cluster_discovery)
     {
-        shared->cluster_discovery = std::make_unique<ClusterDiscovery>(*config, getGlobalContext());
+        shared->cluster_discovery = std::make_unique<ClusterDiscovery>(*config, getGlobalContext(), getMacros());
     }
 
     /// Do not update clusters if this part of config wasn't changed.

tests/integration/test_cluster_discovery/config/macros.xml

@@ -0,0 +1,5 @@
+<clickhouse>
+    <macros>
+        <autocluster>test_auto_cluster</autocluster>
+    </macros>
+</clickhouse>

tests/integration/test_cluster_discovery/test.py

@@ -20,7 +20,7 @@
 nodes = {
     node_name: cluster.add_instance(
         node_name,
-        main_configs=[shard_config, "config/config_discovery_path.xml"],
+        main_configs=[shard_config, "config/config_discovery_path.xml", "config/macros.xml"],
         stay_alive=True,
         with_zookeeper=True,
     )
@@ -122,3 +122,18 @@ def test_cluster_discovery_startup_and_stop(start_cluster):
 
     # cleanup
     nodes["node0"].query("DROP TABLE tbl ON CLUSTER 'test_auto_cluster' SYNC")
+
+
+def test_cluster_discovery_macros(start_cluster):
+    # wait for all nodes to be started
+    check_nodes_count = functools.partial(
+        check_on_cluster, what="count()", msg="Wrong nodes count in cluster"
+    )
+    total_nodes = len(nodes) - 1
+    check_nodes_count([nodes["node_observer"]], total_nodes)
+
+    # check macros
+    res = nodes["node_observer"].query(
+        "SELECT sum(number) FROM clusterAllReplicas('{autocluster}', system.numbers) WHERE number=1"
+    )
+    assert res.strip() == "5"