Merge branch 'master' into delta-kernel-rs-support-local-storage

Author: kssenii

ci/defs/job_configs.py

@@ -867,9 +867,13 @@ class JobConfigs:
         runs_on=RunnerLabels.FUNC_TESTER_AMD,
         command="python3 ./ci/jobs/clickbench.py",
         digest_config=Job.CacheDigestConfig(
-            include_paths=["./ci/jobs/clickbench.py", "./ci/jobs/scripts/clickbench/"],
+            include_paths=[
+                "./ci/jobs/clickbench.py",
+                "./ci/jobs/scripts/clickbench/",
+                "./ci/jobs/scripts/functional_tests/setup_log_cluster.sh",
+            ],
         ),
-        run_in_docker="clickhouse/stateless-test+--shm-size=16g",
+        run_in_docker="clickhouse/stateless-test+--shm-size=16g+--network=host",
     ).parametrize(
         parameter=[
             BuildTypes.AMD_RELEASE,

ci/jobs/build_clickhouse.py

@@ -13,7 +13,7 @@
 temp_dir = f"{current_directory}/ci/tmp"
 
 BUILD_TYPE_TO_CMAKE = {
-    BuildTypes.AMD_DEBUG: f"    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE=          -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE=sccache -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON",
+    BuildTypes.AMD_DEBUG: f"    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE=          -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE=sccache -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DENABLE_XRAY=1",
     BuildTypes.AMD_RELEASE: f"  cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None  -DENABLE_THINLTO=1 -DSANITIZE=          -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE=sccache -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1",
     BuildTypes.AMD_BINARY: f"   cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None  -DENABLE_THINLTO=0 -DSANITIZE=          -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE=sccache -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON",
     BuildTypes.AMD_ASAN: f"     cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None  -DENABLE_THINLTO=0 -DSANITIZE=address   -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE=sccache -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON",

ci/jobs/clickbench.py

@@ -1,7 +1,7 @@
-from praktika.result import Result
-from praktika.utils import Shell, Utils
-
 from ci.jobs.scripts.clickhouse_proc import ClickHouseLight
+from ci.praktika.info import Info
+from ci.praktika.result import Result
+from ci.praktika.utils import Shell, Utils
 
 temp_dir = f"{Utils.cwd()}/ci/tmp/"
 
@@ -16,7 +16,11 @@ def main():
         print("Install ClickHouse")
 
         def install():
-            return ch.install() and ch.clickbench_config_tweaks()
+            return (
+                ch.install()
+                and ch.clickbench_config_tweaks()
+                and ch.create_log_export_config()
+            )
 
         results.append(
             Result.from_commands_run(name="Install ClickHouse", command=install)
@@ -27,15 +31,16 @@ def install():
         print("Start ClickHouse")
 
         def start():
-            return ch.start()
-
-        log_export_config = f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --config-logs-export-cluster {ch.config_path}/config.d/system_logs_export.yaml"
-        setup_logs_replication = f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication"
+            return ch.start() and (
+                ch.start_log_exports(check_start_time=stop_watch.start_time)
+                if not Info().is_local_run
+                else True
+            )
 
         results.append(
             Result.from_commands_run(
                 name="Start ClickHouse",
-                command=[start, log_export_config, setup_logs_replication],
+                command=start,
             )
         )
         res = results[-1].is_ok()
@@ -92,7 +97,9 @@ def start():
         verbose=True,
     )
 
-    Result.create_from(results=results, stopwatch=stop_watch, files=[]).complete_job()
+    Result.create_from(
+        results=results, stopwatch=stop_watch, files=[ch.log_file]
+    ).complete_job()
 
 
 if __name__ == "__main__":

ci/jobs/scripts/clickhouse_proc.py

@@ -3,11 +3,28 @@
 import time
 from pathlib import Path
 
-from praktika.utils import Shell, Utils
+from ci.praktika import Secret
+from ci.praktika.info import Info
+from ci.praktika.utils import Shell, Utils
 
 temp_dir = f"{Utils.cwd()}/ci/tmp"
 
 
+LOG_EXPORT_CONFIG_TEMPLATE = """
+remote_servers:
+    {CLICKHOUSE_CI_LOGS_CLUSTER}:
+        shard:
+            replica:
+                secure: 1
+                user: '{CLICKHOUSE_CI_LOGS_USER}'
+                host: '{CLICKHOUSE_CI_LOGS_HOST}'
+                port: 9440
+                password: '{CLICKHOUSE_CI_LOGS_PASSWORD}'
+"""
+CLICKHOUSE_CI_LOGS_CLUSTER = "system_logs_export"
+CLICKHOUSE_CI_LOGS_USER = "ci"
+
+
 class ClickHouseProc:
     BACKUPS_XML = """
 <clickhouse>
@@ -232,6 +249,49 @@ def fuzzer_config_tweaks(self):
             res = res and Shell.check(command, verbose=True)
         return res
 
+    def create_log_export_config(self):
+        print("Create log export config")
+        config_file = Path(self.config_path) / "config.d" / "system_logs_export.yaml"
+
+        self.log_export_host = Secret.Config(
+            name="clickhouse_ci_logs_host",
+            type=Secret.Type.AWS_SSM_VAR,
+            region="us-east-1",
+        ).get_value()
+
+        self.log_export_password = Secret.Config(
+            name="clickhouse_ci_logs_password",
+            type=Secret.Type.AWS_SSM_VAR,
+            region="us-east-1",
+        ).get_value()
+
+        config_content = LOG_EXPORT_CONFIG_TEMPLATE.format(
+            CLICKHOUSE_CI_LOGS_CLUSTER=CLICKHOUSE_CI_LOGS_CLUSTER,
+            CLICKHOUSE_CI_LOGS_HOST=self.log_export_host,
+            CLICKHOUSE_CI_LOGS_USER=CLICKHOUSE_CI_LOGS_USER,
+            CLICKHOUSE_CI_LOGS_PASSWORD=self.log_export_password,
+        )
+
+        with open(config_file, "w") as f:
+            f.write(config_content)
+
+    def start_log_exports(self, check_start_time):
+        print("Start log export")
+        os.environ["CLICKHOUSE_CI_LOGS_CLUSTER"] = CLICKHOUSE_CI_LOGS_CLUSTER
+        os.environ["CLICKHOUSE_CI_LOGS_HOST"] = self.log_export_host
+        os.environ["CLICKHOUSE_CI_LOGS_USER"] = CLICKHOUSE_CI_LOGS_USER
+        os.environ["CLICKHOUSE_CI_LOGS_PASSWORD"] = self.log_export_password
+        info = Info()
+        os.environ["EXTRA_COLUMNS_EXPRESSION"] = (
+            f"CAST({info.pr_number} AS UInt32) AS pull_request_number, '{info.sha}' AS commit_sha, toDateTime('{Utils.timestamp_to_str(check_start_time)}', 'UTC') AS check_start_time, toLowCardinality('{info.job_name}') AS check_name, toLowCardinality('{info.instance_type}') AS instance_type, '{info.instance_id}' AS instance_id"
+        )
+
+        Shell.check(
+            "./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication",
+            verbose=True,
+            strict=True,
+        )
+
     def start(self):
         print(f"Starting ClickHouse server")
         print("Command: ", self.start_cmd)

ci/jobs/scripts/functional_tests/setup_log_cluster.sh

@@ -9,15 +9,15 @@ set -e
 # Config file contains KEY=VALUE pairs with any necessary parameters like:
 # CLICKHOUSE_CI_LOGS_HOST - remote host
 # CLICKHOUSE_CI_LOGS_USER - password for user
-# CLICKHOUSE_CI_LOGS_PASSWORD - password for user
-CLICKHOUSE_CI_LOGS_CREDENTIALS=${CLICKHOUSE_CI_LOGS_CREDENTIALS:-/tmp/export-logs-config.sh}
-CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
+# CLICKHOUSE_CI_LOGS_PACreate all configured system logsSSWORD - password for user
 
 # Pre-configured destination cluster, where to export the data
 CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
 
+[ -n "$EXTRA_COLUMNS_EXPRESSION" ] || { echo "ERROR: EXTRA_COLUMNS_EXPRESSION env must be defined"; exit 1; }
 EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, INDEX ix_pr (pull_request_number) TYPE set(100), INDEX ix_commit (commit_sha) TYPE set(100), INDEX ix_check_time (check_start_time) TYPE minmax, "}
 EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS UInt32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, toLowCardinality('') AS check_name, toLowCardinality('') AS instance_type, '' AS instance_id"}
+echo "EXTRA_COLUMNS_EXPRESSION=$EXTRA_COLUMNS_EXPRESSION"
 EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name"}
 
 # trace_log needs more columns for symbolization
@@ -57,6 +57,7 @@ function check_logs_credentials
 
     # First check, if all necessary parameters are set
     set +x
+    echo "Check CI Log cluster..."
     for parameter in CLICKHOUSE_CI_LOGS_HOST CLICKHOUSE_CI_LOGS_USER CLICKHOUSE_CI_LOGS_PASSWORD; do
       export -p | grep -q "$parameter" || {
         echo "Credentials parameter $parameter is unset"
@@ -75,51 +76,12 @@ function check_logs_credentials
     fi
 )
 
-function config_logs_export_cluster
-(
-    # The function is launched in a separate shell instance to not expose the
-    # exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
-    set +x
-    if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
-        echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
-        return
-    fi
-    set -a
-    # shellcheck disable=SC1090
-    source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
-    set +a
-    __shadow_credentials
-    echo "Checking if the credentials work"
-    check_logs_credentials || return 0
-    cluster_config="${1:-/etc/clickhouse-server/config.d/system_logs_export.yaml}"
-    mkdir -p "$(dirname "$cluster_config")"
-    echo "remote_servers:
-    ${CLICKHOUSE_CI_LOGS_CLUSTER}:
-        shard:
-            replica:
-                secure: 1
-                user: '${CLICKHOUSE_CI_LOGS_USER}'
-                host: '${CLICKHOUSE_CI_LOGS_HOST}'
-                port: 9440
-                password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
-" > "$cluster_config"
-    echo "Cluster ${CLICKHOUSE_CI_LOGS_CLUSTER} is confugured in ${cluster_config}"
-)
-
 function setup_logs_replication
 (
     # The function is launched in a separate shell instance to not expose the
-    # exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
+    # exported values
     set +x
     # disable output
-    if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
-        echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
-        return 0
-    fi
-    set -a
-    # shellcheck disable=SC1090
-    source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
-    set +a
     __shadow_credentials
     echo "Checking if the credentials work"
     check_logs_credentials || return 0
@@ -246,14 +208,9 @@ while [[ "$#" -gt 0 ]]; do
             echo "Setting up log replication..."
             setup_logs_replication
             ;;
-        --config-logs-export-cluster)
-            echo "Configuring logs export for the cluster..."
-            config_logs_export_cluster "$2"
-            shift
-            ;;
         *)
             echo "Unknown option: $1"
-            echo "Usage: $0 [--stop-log-replication | --setup-logs-replication | --config-logs-export-cluster ]"
+            echo "Usage: $0 [--stop-log-replication | --setup-logs-replication ]"
             exit 1
             ;;
     esac

ci/praktika/secret.py

@@ -15,6 +15,7 @@ class Type:
     class Config:
         name: str
         type: str
+        region: str = ""
 
         def is_gh_secret(self):
             return self.type == Secret.Type.GH_SECRET
@@ -33,8 +34,11 @@ def get_value(self):
                 assert False, f"Not supported secret type, secret [{self}]"
 
         def get_aws_ssm_var(self):
+            region = ""
+            if self.region:
+                region = f" --region {self.region}"
             res = Shell.get_output(
-                f"aws ssm get-parameter --name {self.name} --with-decryption --output text --query Parameter.Value",
+                f"aws ssm get-parameter --name {self.name} --with-decryption --output text --query Parameter.Value {region}",
                 strict=True,
             )
             return res
@@ -43,13 +47,13 @@ def get_aws_ssm_secret(self):
             name, secret_key_name = self.name, ""
             if "." in self.name:
                 name, secret_key_name = self.name.split(".")
-            cmd = f"aws secretsmanager get-secret-value --secret-id  {name} --query SecretString --output text"
+            region = ""
+            if self.region:
+                region = f" --region {self.region}"
+            cmd = f"aws secretsmanager get-secret-value --secret-id  {name} --query SecretString --output text {region}"
             if secret_key_name:
                 cmd += f" | jq -r '.[\"{secret_key_name}\"]'"
-            res = Shell.get_output(cmd, verbose=True)
-            if not res:
-                print(f"ERROR: Failed to get secret [{self.name}]")
-                raise RuntimeError()
+            res = Shell.get_output(cmd, verbose=True, strict=True)
             return res
 
         def get_gh_secret(self):

src/Backups/BackupCoordinationReplicatedTables.cpp

@@ -36,7 +36,7 @@ class BackupCoordinationReplicatedTables::CoveredPartsFinder
     {
         auto new_min_block = new_part_info.min_block;
         auto new_max_block = new_part_info.max_block;
-        auto & parts = partitions[new_part_info.partition_id];
+        auto & parts = partitions[new_part_info.getPartitionId()];
 
         /// Find the first part with max_block >= `part_info.min_block`.
         auto first_it = parts.lower_bound(new_min_block);
@@ -99,7 +99,7 @@ class BackupCoordinationReplicatedTables::CoveredPartsFinder
 
     bool isCoveredByAnotherPart(const MergeTreePartInfo & part_info) const
     {
-        auto partition_it = partitions.find(part_info.partition_id);
+        auto partition_it = partitions.find(part_info.getPartitionId());
         if (partition_it == partitions.end())
             return false;
 
@@ -290,11 +290,12 @@ void BackupCoordinationReplicatedTables::prepare() const
             for (const auto & [part_name, part_replicas] : table_info.replicas_by_part_name)
             {
                 auto part_info = MergeTreePartInfo::fromPartName(part_name, MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING);
+                const auto & partition_id = part_info.getPartitionId();
 
                 auto & min_data_versions_by_partition = table_info.min_data_versions_by_partition;
-                auto it2 = min_data_versions_by_partition.find(part_info.partition_id);
+                auto it2 = min_data_versions_by_partition.find(partition_id);
                 if (it2 == min_data_versions_by_partition.end())
-                    min_data_versions_by_partition[part_info.partition_id] = part_info.getDataVersion();
+                    min_data_versions_by_partition[partition_id] = part_info.getDataVersion();
                 else
                     it2->second = std::min(it2->second, part_info.getDataVersion());
 

src/Backups/RestorerFromBackup.cpp

@@ -485,7 +485,7 @@ void RestorerFromBackup::findTableInBackupImpl(const QualifiedTableName & table_
     res_table_info.has_data = table_has_data;
     res_table_info.data_path_in_backup = data_path_in_backup;
 
-    tables_dependencies.addDependencies(table_name, table_dependencies);
+    tables_dependencies.addDependencies(table_name, table_dependencies.dependencies);
 
     if (partitions)
     {

src/Core/ServerSettings.cpp

@@ -1057,6 +1057,7 @@ The policy on how to perform a scheduling of CPU slots specified by `concurrent_
     ```
     )", 0) \
     DECLARE(Bool, storage_shared_set_join_use_inner_uuid, true, "If enabled, an inner UUID is generated during the creation of SharedSet and SharedJoin. ClickHouse Cloud only", 0) \
+    DECLARE(UInt64, startup_mv_delay_ms, 0, R"(Debug parameter to simulate materizlied view creation delay)", 0) \
     DECLARE(UInt64, os_cpu_busy_time_threshold, 1'000'000, "Threshold of OS CPU busy time in microseconds (OSCPUVirtualTimeMicroseconds metric) to consider CPU doing some useful work, no CPU overload would be considered if busy time was below this value.", 0) \
     DECLARE(Float, min_os_cpu_wait_time_ratio_to_drop_connection, 10.0, "Min ratio between OS CPU wait (OSCPUWaitMicroseconds metric) and busy (OSCPUVirtualTimeMicroseconds metric) times to consider dropping connections. Linear interpolation between min and max ratio is used to calculate the probability, the probability is 0 at this point.", 0) \
     DECLARE(Float, max_os_cpu_wait_time_ratio_to_drop_connection, 20.0, "Max ratio between OS CPU wait (OSCPUWaitMicroseconds metric) and busy (OSCPUVirtualTimeMicroseconds metric) times to consider dropping connections. Linear interpolation between min and max ratio is used to calculate the probability, the probability is 1 at this point.", 0) \