Merge pull request ClickHouse#79670 from ClickHouse/hanfei/refine-keeper-oom-exception

refine the exception message when keeper reaches soft memory limit

Author: hanfei1991

src/Common/ZooKeeper/IKeeper.cpp

@@ -80,6 +80,7 @@ const char * errorMessage(Error code)
         case Error::ZOPERATIONTIMEOUT:        return "Operation timeout";
         case Error::ZBADARGUMENTS:            return "Bad arguments";
         case Error::ZINVALIDSTATE:            return "Invalid zhandle state";
+        case Error::ZOUTOFMEMORY:             return "Out of Memory";
         case Error::ZAPIERROR:                return "API error";
         case Error::ZNONODE:                  return "No node";
         case Error::ZNOAUTH:                  return "Not authenticated";

src/Common/ZooKeeper/IKeeper.h

@@ -82,6 +82,7 @@ enum class Error : int32_t
     ZOPERATIONTIMEOUT = -7,     /// Operation timeout
     ZBADARGUMENTS = -8,         /// Invalid arguments
     ZINVALIDSTATE = -9,         /// Invalid zhandle state
+    ZOUTOFMEMORY = -10,         /// Keeper has reached soft memory limit
 
     /** API errors.
         * This is never thrown by the server, it shouldn't be used other than

src/Coordination/KeeperDispatcher.cpp

@@ -173,7 +173,7 @@ void KeeperDispatcher::requestThread()
                         ReadableSize(total_memory_tracker.get()),
                         ReadableSize(total_memory_tracker.getRSS()),
                         request.request->getOpNum());
-                    addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS);
+                    addErrorResponses({request}, Coordination::Error::ZOUTOFMEMORY);
                     continue;
                 }
 

tests/integration/test_keeper_memory_soft_limit/configs/keeper_config1.xml

@@ -15,7 +15,7 @@
             <value>az-zoo1</value>
         </availability_zone>
         <server_id>1</server_id>
-        <max_memory_usage_soft_limit>200000000</max_memory_usage_soft_limit>
+        <max_memory_usage_soft_limit>80000000</max_memory_usage_soft_limit>
 
         <coordination_settings>
             <operation_timeout_ms>10000</operation_timeout_ms>

tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml

@@ -16,7 +16,7 @@
             <value>az-zoo2</value>
             <enable_auto_detection_on_cloud>1</enable_auto_detection_on_cloud>
         </availability_zone>
-        <max_memory_usage_soft_limit>200000000</max_memory_usage_soft_limit>
+        <max_memory_usage_soft_limit>80000000</max_memory_usage_soft_limit>
 
         <coordination_settings>
             <operation_timeout_ms>10000</operation_timeout_ms>

tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml

@@ -13,7 +13,7 @@
         <tcp_port>2181</tcp_port>
         <server_id>3</server_id>
 
-        <max_memory_usage_soft_limit>200000000</max_memory_usage_soft_limit>
+        <max_memory_usage_soft_limit>80000000</max_memory_usage_soft_limit>
 
         <coordination_settings>
             <operation_timeout_ms>10000</operation_timeout_ms>

tests/integration/test_keeper_memory_soft_limit/configs/setting.xml

@@ -0,0 +1,3 @@
+<clickhouse>
+<allow_zookeeper_write>1</allow_zookeeper_write>
+</clickhouse>

tests/integration/test_keeper_memory_soft_limit/test.py

@@ -4,8 +4,6 @@
 
 import pytest
 from kazoo.client import KazooClient
-from kazoo.exceptions import ConnectionLoss
-
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__, keeper_config_dir="configs/")
@@ -16,6 +14,7 @@
     stay_alive=True,
     with_zookeeper=True,
     with_remote_database_disk=False,  # Disable `with_remote_database_disk` as the test does not use the default Keeper.
+    main_configs=["configs/setting.xml"],
 )
 
 
@@ -44,23 +43,28 @@ def started_cluster():
 
 
 def test_soft_limit_create(started_cluster):
+    if node.is_built_with_sanitizer():
+        pytest.skip("Disabled for sanitizers")
     started_cluster.wait_zookeeper_to_start()
+    node_zk = get_connection_zk("zoo1")
     try:
-        node_zk = get_connection_zk("zoo1")
         loop_time = 100000
         node_zk.create("/test_soft_limit", b"abc")
+        path = "/test_soft_limit"
 
         for i in range(loop_time):
-            node_zk.create(
-                "/test_soft_limit/node_" + str(i), random_string(1000).encode()
-            )
-    except ConnectionLoss:
+            name = "node_" + str(i)
+            node.query(f"INSERT INTO system.zookeeper (name, path, value) values ('{name}', '{path}', repeat('a', 3000))")
+    except Exception as e:
+        # the message contains out of memory so the users will not be confused.
+        assert 'out of memory' in str(e).lower()
         txn = node_zk.transaction()
         for i in range(10):
             txn.delete("/test_soft_limit/node_" + str(i))
 
         txn.create("/test_soft_limit/node_1000001" + str(i), b"abcde")
         txn.commit()
+        assert "0\n"  == node.query("select sum(ProfileEvent_ZooKeeperHardwareExceptions) from system.metric_log")
         return
 
     raise Exception("all records are inserted but no error occurs")