Merge pull request ClickHouse#79296 from Blargian/ci_mod

Blargian · web-flow · commit c6dc04b2034b · 2025-04-22T22:18:16.000Z
CI: improve subtask output in CI report
diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py
@@ -156,7 +156,6 @@ def main():
                 name="Cmake configuration",
                 command=cmake_cmd,
                 workdir=build_dir,
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -177,7 +176,6 @@ def main():
                 name="Build ClickHouse",
                 command=f"ninja {targets}",
                 workdir=build_dir,
-                with_log=True,
             )
         )
         run_shell("sccache stats", "sccache --show-stats")
@@ -208,7 +206,6 @@ def main():
                     f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={temp_dir} BUILD_TYPE={BUILD_TYPE_TO_DEB_PACKAGE_TYPE[build_type]} VERSION_STRING={version_dict['string']} DEB_ARCH={deb_arch} ./build --deb {'--rpm --tgz' if 'release' in build_type else ''}",
                 ],
                 workdir=build_dir,
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/clickbench.py b/ci/jobs/clickbench.py
@@ -36,7 +36,6 @@ def start():
             Result.from_commands_run(
                 name="Start ClickHouse",
                 command=[start, log_export_config, setup_logs_replication],
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/collect_statistics.py b/ci/jobs/collect_statistics.py
@@ -139,9 +139,7 @@ def get_all_job_names():
         return True
 
     results.append(
-        Result.from_commands_run(
-            name="Get all job names", command=get_all_job_names, with_info=True
-        )
+        Result.from_commands_run(name="Get all job names", command=get_all_job_names)
     )
     if not results[-1].is_ok():
         sys.exit()
diff --git a/ci/jobs/docker_server_job.py b/ci/jobs/docker_server_job.py
@@ -103,11 +103,7 @@ def main():
     command = f"docker buildx build --platform {platforms} \
                     -t {image_repo}:tmp {image_path} -f {docker_path}/Dockerfile.ubuntu"
 
-    results.append(
-        Result.from_commands_run(
-            name=f"{image_repo}:tmp", command=command, with_log=True
-        )
-    )
+    results.append(Result.from_commands_run(name=f"{image_repo}:tmp", command=command))
 
     Result.create_from(results=results, stopwatch=stopwatch).complete_job()
 
diff --git a/ci/jobs/fast_test.py b/ci/jobs/fast_test.py
@@ -175,7 +175,6 @@ def main():
                 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 -DENABLE_NURAFT=1 -DENABLE_SIMDJSON=1 \
                 -DENABLE_JEMALLOC=1 -DENABLE_LIBURING=1 -DENABLE_YAML_CPP=1 -DCOMPILER_CACHE=sccache",
                 workdir=build_dir,
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -187,7 +186,6 @@ def main():
                 name="Build ClickHouse",
                 command="ninja clickhouse-bundle clickhouse-stripped",
                 workdir=build_dir,
-                with_log=True,
             )
         )
         Shell.check("sccache --show-stats")
@@ -205,7 +203,6 @@ def main():
                 name="Check and Compress binary",
                 command=commands,
                 workdir=build_dir,
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -223,7 +220,6 @@ def main():
             Result.from_commands_run(
                 name="Install ClickHouse Config",
                 command=commands,
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/functional_stateful_tests.py b/ci/jobs/functional_stateful_tests.py
@@ -104,9 +104,7 @@ def main():
             f"clickhouse-server --version",
         ]
         results.append(
-            Result.from_commands_run(
-                name="Install ClickHouse", command=commands, with_log=True
-            )
+            Result.from_commands_run(name="Install ClickHouse", command=commands)
         )
         res = results[-1].is_ok()
 
diff --git a/ci/jobs/functional_stateless_tests.py b/ci/jobs/functional_stateless_tests.py
@@ -121,9 +121,7 @@ def main():
             f"clickhouse-server --version",
         ]
         results.append(
-            Result.from_commands_run(
-                name="Install ClickHouse", command=commands, with_log=True
-            )
+            Result.from_commands_run(name="Install ClickHouse", command=commands)
         )
         res = results[-1].is_ok()
 
diff --git a/ci/jobs/fuzzers_job.py b/ci/jobs/fuzzers_job.py
@@ -38,7 +38,6 @@ def start():
             Result.from_commands_run(
                 name="Start ClickHouse",
                 command=[start, log_export_config, setup_logs_replication],
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -51,7 +50,6 @@ def start():
             Result.from_commands_run(
                 name="Start ClickHouse",
                 command=commands,
-                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/performance_tests.py b/ci/jobs/performance_tests.py
@@ -331,9 +331,7 @@ def main():
             "clickhouse-local --version",
         ]
         results.append(
-            Result.from_commands_run(
-                name="Install ClickHouse", command=commands, with_log=True
-            )
+            Result.from_commands_run(name="Install ClickHouse", command=commands)
         )
         res = results[-1].is_ok()
 
@@ -352,7 +350,7 @@ def main():
             ]
             results.append(
                 Result.from_commands_run(
-                    name="Install Reference ClickHouse", command=commands, with_log=True
+                    name="Install Reference ClickHouse", command=commands
                 )
             )
             res = results[-1].is_ok()
@@ -421,9 +419,7 @@ def restart_ch():
             f"cp -R {temp_dir}/coordination0 {perf_left}/coordination",
             f"cp -R {temp_dir}/coordination0 {perf_right}/coordination",
         ]
-        results.append(
-            Result.from_commands_run(name="Configure", command=commands, with_log=True)
-        )
+        results.append(Result.from_commands_run(name="Configure", command=commands))
         res = results[-1].is_ok()
 
     leftCH = CHServer(is_left=True)
@@ -524,7 +520,6 @@ def run_tests():
             Result.from_commands_run(
                 name="Report",
                 command=commands,
-                with_log=True,
                 workdir=perf_wd,
             )
         )
diff --git a/ci/praktika/docker.py b/ci/praktika/docker.py
@@ -59,9 +59,7 @@ def build(cls, config: "Docker.Config", digests, amd_only, arm_only, with_log):
 
             command = f"docker buildx build --builder default {tags_substr} {from_tag} --platform {','.join(platforms)} --cache-to type=inline --cache-from type=registry,ref={config.name} {config.path} --push"
 
-            return Result.from_commands_run(
-                name=name, command=command, with_info=with_log
-            )
+            return Result.from_commands_run(name=name, command=command)
         else:
             return Result(
                 name=name,
@@ -105,7 +103,6 @@ def merge_manifest(
         return Result.from_commands_run(
             name=f"merge: {config.name}:{digests[config.name]} (latest={add_latest})",
             command=commands,
-            with_info=with_log,
             fail_fast=True,
         )
 
diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py
@@ -186,7 +186,6 @@ def _check_yaml_up_to_date():
         return Result.from_commands_run(
             name="Check Workflows",
             command=commands,
-            with_info=True,
             fail_fast=True,
         )
 
@@ -269,9 +268,7 @@ def _check_db(workflow):
                 name = pre_check.__name__
             else:
                 name = str(pre_check)
-            res_.append(
-                Result.from_commands_run(name=name, command=pre_check, with_info=True)
-            )
+            res_.append(Result.from_commands_run(name=name, command=pre_check))
 
         results.append(
             Result.create_from(name="Pre Hooks", results=res_, stopwatch=sw_)
@@ -428,7 +425,6 @@ def check_affected_jobs():
             Result.from_commands_run(
                 name="Filter not affected jobs",
                 command=check_affected_jobs,
-                with_info=True,
             )
         )
 
@@ -497,9 +493,7 @@ def _finish_workflow(workflow, job_name):
                 name = check.__name__
             else:
                 name = str(check)
-            results_.append(
-                Result.from_commands_run(name=name, command=check, with_info=True)
-            )
+            results_.append(Result.from_commands_run(name=name, command=check))
 
         results.append(
             Result.create_from(name="Post Hooks", results=results_, stopwatch=sw_)
diff --git a/ci/praktika/result.py b/ci/praktika/result.py
@@ -386,6 +386,7 @@ def from_commands_run(
         command,
         with_log=False,
         with_info=False,
+        with_info_on_failure=True,
         fail_fast=True,
         workdir=None,
         command_args=None,
@@ -399,6 +400,7 @@ def from_commands_run(
         :param workdir: Optional working directory.
         :param with_log: Boolean flag to log output to a file.
         :param with_info: Fill in Result.info from command output
+        :param with_info_on_failure: Fill in Result.info from command output on failure only
         :param fail_fast: Boolean flag to stop execution if one command fails.
         :param command_args: Positional arguments for the callable command.
         :param command_kwargs: Keyword arguments for the callable command.
@@ -413,7 +415,7 @@ def from_commands_run(
         # Set log file path if logging is enabled
         if with_log:
             log_file = f"{Utils.absolute_path(Settings.TEMP_DIR)}/{Utils.normalize_string(name)}.log"
-        elif with_info:
+        elif with_info or with_info_on_failure:
             log_file = f"/tmp/praktika_{Utils.normalize_string(name)}.log"
         else:
             log_file = None
@@ -430,22 +432,22 @@ def from_commands_run(
             for command_ in command:
                 if callable(command_):
                     # If command is a Python function, call it with provided arguments
-                    if with_info:
+                    if with_info or with_info_on_failure:
                         buffer = io.StringIO()
                         with redirect_stdout(buffer):
                             result = command_(*command_args, **command_kwargs)
-                        error_infos = buffer.getvalue()
                     else:
                         result = command_(*command_args, **command_kwargs)
-                    if isinstance(result, bool):
-                        res = result
-                    elif result:
-                        error_infos.append(str(result))
-                        res = False
+                    res = result if isinstance(result, bool) else not bool(result)
+                    if (with_info_on_failure and not res) or with_info:
+                        if isinstance(result, bool):
+                            error_infos = buffer.getvalue()
+                        else:
+                            error_infos = str(result).splitlines()
                 else:
                     # Run shell command in a specified directory with logging and verbosity
                     exit_code = Shell.run(command_, verbose=True, log_file=log_file)
-                    if with_info:
+                    if with_info or (with_info_on_failure and exit_code != 0):
                         with open(log_file, "r") as f:
                             error_infos.append(f.read().strip())
                     res = exit_code == 0
@@ -456,11 +458,17 @@ def from_commands_run(
                     break
 
         # Create and return the result object with status and log file (if any)
+        MAX_LINES_IN_INFO = 100
         return Result.create_from(
             name=name,
             status=res,
             stopwatch=stop_watch_,
-            info=error_infos,
+            info=(
+                error_infos
+                if len(error_infos) < MAX_LINES_IN_INFO
+                else [f" ~~~ truncated {len(error_infos)-MAX_LINES_IN_INFO} lines ~~~"]
+                + error_infos[-MAX_LINES_IN_INFO:]
+            ),
             files=[log_file] if with_log else None,
         )
 
diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py
@@ -373,9 +373,7 @@ def _post_run(
                     name = check.__name__
                 else:
                     name = str(check)
-                results_.append(
-                    Result.from_commands_run(name=name, command=check, with_info=True)
-                )
+                results_.append(Result.from_commands_run(name=name, command=check))
             result.results.append(
                 Result.create_from(name="Post Hooks", results=results_, stopwatch=sw_)
             )
diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md
@@ -892,7 +892,7 @@ SELECT countEqual([1, 2, NULL, NULL], NULL)
 
 Returns the array \[1, 2, 3, ..., length (arr) \]
 
-This function is normally used with ARRAY JOIN. It allows counting something just once for each array after applying ARRAY JOIN. Example:
+This function is normally used with `ARRAY JOIN`. It allows counting something just once for each array after applying `ARRAY JOIN`. Example:
 
 ```sql
 SELECT

Original file line number	Diff line number	Diff line change
`@@ -156,7 +156,6 @@ def main():`
`156`	`156`	`name="Cmake configuration",`
`157`	`157`	`command=cmake_cmd,`
`158`	`158`	`workdir=build_dir,`
`159`		`- with_log=True,`
`160`	`159`	`)`
`161`	`160`	`)`
`162`	`161`	`res = results[-1].is_ok()`
`@@ -177,7 +176,6 @@ def main():`
`177`	`176`	`name="Build ClickHouse",`
`178`	`177`	`command=f"ninja {targets}",`
`179`	`178`	`workdir=build_dir,`
`180`		`- with_log=True,`
`181`	`179`	`)`
`182`	`180`	`)`
`183`	`181`	`run_shell("sccache stats", "sccache --show-stats")`
`@@ -208,7 +206,6 @@ def main():`
`208`	`206`	`f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={temp_dir} BUILD_TYPE={BUILD_TYPE_TO_DEB_PACKAGE_TYPE[build_type]} VERSION_STRING={version_dict['string']} DEB_ARCH={deb_arch} ./build --deb {'--rpm --tgz' if 'release' in build_type else ''}",`
`209`	`207`	`],`
`210`	`208`	`workdir=build_dir,`
`211`		`- with_log=True,`
`212`	`209`	`)`
`213`	`210`	`)`
`214`	`211`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,6 @@ def start():`
`36`	`36`	`Result.from_commands_run(`
`37`	`37`	`name="Start ClickHouse",`
`38`	`38`	`command=[start, log_export_config, setup_logs_replication],`
`39`		`- with_log=True,`
`40`	`39`	`)`
`41`	`40`	`)`
`42`	`41`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -175,7 +175,6 @@ def main():`
`175`	`175`	`-DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 -DENABLE_NURAFT=1 -DENABLE_SIMDJSON=1 \`
`176`	`176`	`-DENABLE_JEMALLOC=1 -DENABLE_LIBURING=1 -DENABLE_YAML_CPP=1 -DCOMPILER_CACHE=sccache",`
`177`	`177`	`workdir=build_dir,`
`178`		`- with_log=True,`
`179`	`178`	`)`
`180`	`179`	`)`
`181`	`180`	`res = results[-1].is_ok()`
`@@ -187,7 +186,6 @@ def main():`
`187`	`186`	`name="Build ClickHouse",`
`188`	`187`	`command="ninja clickhouse-bundle clickhouse-stripped",`
`189`	`188`	`workdir=build_dir,`
`190`		`- with_log=True,`
`191`	`189`	`)`
`192`	`190`	`)`
`193`	`191`	`Shell.check("sccache --show-stats")`
`@@ -205,7 +203,6 @@ def main():`
`205`	`203`	`name="Check and Compress binary",`
`206`	`204`	`command=commands,`
`207`	`205`	`workdir=build_dir,`
`208`		`- with_log=True,`
`209`	`206`	`)`
`210`	`207`	`)`
`211`	`208`	`res = results[-1].is_ok()`
`@@ -223,7 +220,6 @@ def main():`
`223`	`220`	`Result.from_commands_run(`
`224`	`221`	`name="Install ClickHouse Config",`
`225`	`222`	`command=commands,`
`226`		`- with_log=True,`
`227`	`223`	`)`
`228`	`224`	`)`
`229`	`225`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -104,9 +104,7 @@ def main():`
`104`	`104`	`f"clickhouse-server --version",`
`105`	`105`	`]`
`106`	`106`	`results.append(`
`107`		`- Result.from_commands_run(`
`108`		`- name="Install ClickHouse", command=commands, with_log=True`
`109`		`- )`
	`107`	`+ Result.from_commands_run(name="Install ClickHouse", command=commands)`
`110`	`108`	`)`
`111`	`109`	`res = results[-1].is_ok()`
`112`	`110`
Original file line number	Diff line number	Diff line change
`@@ -121,9 +121,7 @@ def main():`
`121`	`121`	`f"clickhouse-server --version",`
`122`	`122`	`]`
`123`	`123`	`results.append(`
`124`		`- Result.from_commands_run(`
`125`		`- name="Install ClickHouse", command=commands, with_log=True`
`126`		`- )`
	`124`	`+ Result.from_commands_run(name="Install ClickHouse", command=commands)`
`127`	`125`	`)`
`128`	`126`	`res = results[-1].is_ok()`
`129`	`127`
Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,6 @@ def start():`
`38`	`38`	`Result.from_commands_run(`
`39`	`39`	`name="Start ClickHouse",`
`40`	`40`	`command=[start, log_export_config, setup_logs_replication],`
`41`		`- with_log=True,`
`42`	`41`	`)`
`43`	`42`	`)`
`44`	`43`	`res = results[-1].is_ok()`
`@@ -51,7 +50,6 @@ def start():`
`51`	`50`	`Result.from_commands_run(`
`52`	`51`	`name="Start ClickHouse",`
`53`	`52`	`command=commands,`
`54`		`- with_log=True,`
`55`	`53`	`)`
`56`	`54`	`)`
`57`	`55`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -331,9 +331,7 @@ def main():`
`331`	`331`	`"clickhouse-local --version",`
`332`	`332`	`]`
`333`	`333`	`results.append(`
`334`		`- Result.from_commands_run(`
`335`		`- name="Install ClickHouse", command=commands, with_log=True`
`336`		`- )`
	`334`	`+ Result.from_commands_run(name="Install ClickHouse", command=commands)`
`337`	`335`	`)`
`338`	`336`	`res = results[-1].is_ok()`
`339`	`337`
`@@ -352,7 +350,7 @@ def main():`
`352`	`350`	`]`
`353`	`351`	`results.append(`
`354`	`352`	`Result.from_commands_run(`
`355`		`- name="Install Reference ClickHouse", command=commands, with_log=True`
	`353`	`+ name="Install Reference ClickHouse", command=commands`
`356`	`354`	`)`
`357`	`355`	`)`
`358`	`356`	`res = results[-1].is_ok()`
`@@ -421,9 +419,7 @@ def restart_ch():`
`421`	`419`	`f"cp -R {temp_dir}/coordination0 {perf_left}/coordination",`
`422`	`420`	`f"cp -R {temp_dir}/coordination0 {perf_right}/coordination",`
`423`	`421`	`]`
`424`		`- results.append(`
`425`		`- Result.from_commands_run(name="Configure", command=commands, with_log=True)`
`426`		`- )`
	`422`	`+ results.append(Result.from_commands_run(name="Configure", command=commands))`
`427`	`423`	`res = results[-1].is_ok()`
`428`	`424`
`429`	`425`	`leftCH = CHServer(is_left=True)`
`@@ -524,7 +520,6 @@ def run_tests():`
`524`	`520`	`Result.from_commands_run(`
`525`	`521`	`name="Report",`
`526`	`522`	`command=commands,`
`527`		`- with_log=True,`
`528`	`523`	`workdir=perf_wd,`
`529`	`524`	`)`
`530`	`525`	`)`