Merge pull request ClickHouse#79248 from ClickHouse/ci_fix_log_export_in_clickbench

CI: Fix log export in clickbench

Author: maxknv

ci/defs/job_configs.py

@@ -867,9 +867,13 @@ class JobConfigs:
         runs_on=RunnerLabels.FUNC_TESTER_AMD,
         command="python3 ./ci/jobs/clickbench.py",
         digest_config=Job.CacheDigestConfig(
-            include_paths=["./ci/jobs/clickbench.py", "./ci/jobs/scripts/clickbench/"],
+            include_paths=[
+                "./ci/jobs/clickbench.py",
+                "./ci/jobs/scripts/clickbench/",
+                "./ci/jobs/scripts/functional_tests/setup_log_cluster.sh",
+            ],
         ),
-        run_in_docker="clickhouse/stateless-test+--shm-size=16g",
+        run_in_docker="clickhouse/stateless-test+--shm-size=16g+--network=host",
     ).parametrize(
         parameter=[
             BuildTypes.AMD_RELEASE,

ci/jobs/clickbench.py

@@ -1,7 +1,7 @@
-from praktika.result import Result
-from praktika.utils import Shell, Utils
-
 from ci.jobs.scripts.clickhouse_proc import ClickHouseLight
+from ci.praktika.info import Info
+from ci.praktika.result import Result
+from ci.praktika.utils import Shell, Utils
 
 temp_dir = f"{Utils.cwd()}/ci/tmp/"
 
@@ -16,7 +16,11 @@ def main():
         print("Install ClickHouse")
 
         def install():
-            return ch.install() and ch.clickbench_config_tweaks()
+            return (
+                ch.install()
+                and ch.clickbench_config_tweaks()
+                and ch.create_log_export_config()
+            )
 
         results.append(
             Result.from_commands_run(name="Install ClickHouse", command=install)
@@ -27,15 +31,16 @@ def install():
         print("Start ClickHouse")
 
         def start():
-            return ch.start()
-
-        log_export_config = f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --config-logs-export-cluster {ch.config_path}/config.d/system_logs_export.yaml"
-        setup_logs_replication = f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication"
+            return ch.start() and (
+                ch.start_log_exports(check_start_time=stop_watch.start_time)
+                if not Info().is_local_run
+                else True
+            )
 
         results.append(
             Result.from_commands_run(
                 name="Start ClickHouse",
-                command=[start, log_export_config, setup_logs_replication],
+                command=start,
             )
         )
         res = results[-1].is_ok()
@@ -92,7 +97,9 @@ def start():
         verbose=True,
     )
 
-    Result.create_from(results=results, stopwatch=stop_watch, files=[]).complete_job()
+    Result.create_from(
+        results=results, stopwatch=stop_watch, files=[ch.log_file]
+    ).complete_job()
 
 
 if __name__ == "__main__":

ci/jobs/scripts/clickhouse_proc.py

@@ -3,11 +3,28 @@
 import time
 from pathlib import Path
 
-from praktika.utils import Shell, Utils
+from ci.praktika import Secret
+from ci.praktika.info import Info
+from ci.praktika.utils import Shell, Utils
 
 temp_dir = f"{Utils.cwd()}/ci/tmp"
 
 
+LOG_EXPORT_CONFIG_TEMPLATE = """
+remote_servers:
+    {CLICKHOUSE_CI_LOGS_CLUSTER}:
+        shard:
+            replica:
+                secure: 1
+                user: '{CLICKHOUSE_CI_LOGS_USER}'
+                host: '{CLICKHOUSE_CI_LOGS_HOST}'
+                port: 9440
+                password: '{CLICKHOUSE_CI_LOGS_PASSWORD}'
+"""
+CLICKHOUSE_CI_LOGS_CLUSTER = "system_logs_export"
+CLICKHOUSE_CI_LOGS_USER = "ci"
+
+
 class ClickHouseProc:
     BACKUPS_XML = """
 <clickhouse>
@@ -232,6 +249,49 @@ def fuzzer_config_tweaks(self):
             res = res and Shell.check(command, verbose=True)
         return res
 
+    def create_log_export_config(self):
+        print("Create log export config")
+        config_file = Path(self.config_path) / "config.d" / "system_logs_export.yaml"
+
+        self.log_export_host = Secret.Config(
+            name="clickhouse_ci_logs_host",
+            type=Secret.Type.AWS_SSM_VAR,
+            region="us-east-1",
+        ).get_value()
+
+        self.log_export_password = Secret.Config(
+            name="clickhouse_ci_logs_password",
+            type=Secret.Type.AWS_SSM_VAR,
+            region="us-east-1",
+        ).get_value()
+
+        config_content = LOG_EXPORT_CONFIG_TEMPLATE.format(
+            CLICKHOUSE_CI_LOGS_CLUSTER=CLICKHOUSE_CI_LOGS_CLUSTER,
+            CLICKHOUSE_CI_LOGS_HOST=self.log_export_host,
+            CLICKHOUSE_CI_LOGS_USER=CLICKHOUSE_CI_LOGS_USER,
+            CLICKHOUSE_CI_LOGS_PASSWORD=self.log_export_password,
+        )
+
+        with open(config_file, "w") as f:
+            f.write(config_content)
+
+    def start_log_exports(self, check_start_time):
+        print("Start log export")
+        os.environ["CLICKHOUSE_CI_LOGS_CLUSTER"] = CLICKHOUSE_CI_LOGS_CLUSTER
+        os.environ["CLICKHOUSE_CI_LOGS_HOST"] = self.log_export_host
+        os.environ["CLICKHOUSE_CI_LOGS_USER"] = CLICKHOUSE_CI_LOGS_USER
+        os.environ["CLICKHOUSE_CI_LOGS_PASSWORD"] = self.log_export_password
+        info = Info()
+        os.environ["EXTRA_COLUMNS_EXPRESSION"] = (
+            f"CAST({info.pr_number} AS UInt32) AS pull_request_number, '{info.sha}' AS commit_sha, toDateTime('{Utils.timestamp_to_str(check_start_time)}', 'UTC') AS check_start_time, toLowCardinality('{info.job_name}') AS check_name, toLowCardinality('{info.instance_type}') AS instance_type, '{info.instance_id}' AS instance_id"
+        )
+
+        Shell.check(
+            "./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication",
+            verbose=True,
+            strict=True,
+        )
+
     def start(self):
         print(f"Starting ClickHouse server")
         print("Command: ", self.start_cmd)

ci/jobs/scripts/functional_tests/setup_log_cluster.sh

@@ -9,15 +9,15 @@ set -e
 # Config file contains KEY=VALUE pairs with any necessary parameters like:
 # CLICKHOUSE_CI_LOGS_HOST - remote host
 # CLICKHOUSE_CI_LOGS_USER - password for user
-# CLICKHOUSE_CI_LOGS_PASSWORD - password for user
-CLICKHOUSE_CI_LOGS_CREDENTIALS=${CLICKHOUSE_CI_LOGS_CREDENTIALS:-/tmp/export-logs-config.sh}
-CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
+# CLICKHOUSE_CI_LOGS_PACreate all configured system logsSSWORD - password for user
 
 # Pre-configured destination cluster, where to export the data
 CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
 
+[ -n "$EXTRA_COLUMNS_EXPRESSION" ] || { echo "ERROR: EXTRA_COLUMNS_EXPRESSION env must be defined"; exit 1; }
 EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, INDEX ix_pr (pull_request_number) TYPE set(100), INDEX ix_commit (commit_sha) TYPE set(100), INDEX ix_check_time (check_start_time) TYPE minmax, "}
 EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS UInt32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, toLowCardinality('') AS check_name, toLowCardinality('') AS instance_type, '' AS instance_id"}
+echo "EXTRA_COLUMNS_EXPRESSION=$EXTRA_COLUMNS_EXPRESSION"
 EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name"}
 
 # trace_log needs more columns for symbolization
@@ -57,6 +57,7 @@ function check_logs_credentials
 
     # First check, if all necessary parameters are set
     set +x
+    echo "Check CI Log cluster..."
     for parameter in CLICKHOUSE_CI_LOGS_HOST CLICKHOUSE_CI_LOGS_USER CLICKHOUSE_CI_LOGS_PASSWORD; do
       export -p | grep -q "$parameter" || {
         echo "Credentials parameter $parameter is unset"
@@ -75,51 +76,12 @@ function check_logs_credentials
     fi
 )
 
-function config_logs_export_cluster
-(
-    # The function is launched in a separate shell instance to not expose the
-    # exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
-    set +x
-    if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
-        echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
-        return
-    fi
-    set -a
-    # shellcheck disable=SC1090
-    source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
-    set +a
-    __shadow_credentials
-    echo "Checking if the credentials work"
-    check_logs_credentials || return 0
-    cluster_config="${1:-/etc/clickhouse-server/config.d/system_logs_export.yaml}"
-    mkdir -p "$(dirname "$cluster_config")"
-    echo "remote_servers:
-    ${CLICKHOUSE_CI_LOGS_CLUSTER}:
-        shard:
-            replica:
-                secure: 1
-                user: '${CLICKHOUSE_CI_LOGS_USER}'
-                host: '${CLICKHOUSE_CI_LOGS_HOST}'
-                port: 9440
-                password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
-" > "$cluster_config"
-    echo "Cluster ${CLICKHOUSE_CI_LOGS_CLUSTER} is confugured in ${cluster_config}"
-)
-
 function setup_logs_replication
 (
     # The function is launched in a separate shell instance to not expose the
-    # exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
+    # exported values
     set +x
     # disable output
-    if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
-        echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
-        return 0
-    fi
-    set -a
-    # shellcheck disable=SC1090
-    source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
-    set +a
     __shadow_credentials
     echo "Checking if the credentials work"
     check_logs_credentials || return 0
@@ -246,14 +208,9 @@ while [[ "$#" -gt 0 ]]; do
             echo "Setting up log replication..."
             setup_logs_replication
             ;;
-        --config-logs-export-cluster)
-            echo "Configuring logs export for the cluster..."
-            config_logs_export_cluster "$2"
-            shift
-            ;;
         *)
             echo "Unknown option: $1"
-            echo "Usage: $0 [--stop-log-replication | --setup-logs-replication | --config-logs-export-cluster ]"
+            echo "Usage: $0 [--stop-log-replication | --setup-logs-replication ]"
             exit 1
             ;;
     esac

ci/praktika/secret.py

@@ -15,6 +15,7 @@ class Type:
     class Config:
         name: str
         type: str
+        region: str = ""
 
         def is_gh_secret(self):
             return self.type == Secret.Type.GH_SECRET
@@ -33,8 +34,11 @@ def get_value(self):
                 assert False, f"Not supported secret type, secret [{self}]"
 
         def get_aws_ssm_var(self):
+            region = ""
+            if self.region:
+                region = f" --region {self.region}"
             res = Shell.get_output(
-                f"aws ssm get-parameter --name {self.name} --with-decryption --output text --query Parameter.Value",
+                f"aws ssm get-parameter --name {self.name} --with-decryption --output text --query Parameter.Value {region}",
                 strict=True,
             )
             return res
@@ -43,13 +47,13 @@ def get_aws_ssm_secret(self):
             name, secret_key_name = self.name, ""
             if "." in self.name:
                 name, secret_key_name = self.name.split(".")
-            cmd = f"aws secretsmanager get-secret-value --secret-id  {name} --query SecretString --output text"
+            region = ""
+            if self.region:
+                region = f" --region {self.region}"
+            cmd = f"aws secretsmanager get-secret-value --secret-id  {name} --query SecretString --output text {region}"
             if secret_key_name:
                 cmd += f" | jq -r '.[\"{secret_key_name}\"]'"
-            res = Shell.get_output(cmd, verbose=True)
-            if not res:
-                print(f"ERROR: Failed to get secret [{self.name}]")
-                raise RuntimeError()
+            res = Shell.get_output(cmd, verbose=True, strict=True)
             return res
 
         def get_gh_secret(self):