Merge pull request ClickHouse#79205 from ClickHouse/revert-79176-ci_improve_subtasks_output_in_ci_report

maxknv · web-flow · commit 0be0798665da · 2025-04-15T14:31:55.000Z
Revert "CI: Improve subtask output in CI report"
diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py
@@ -156,6 +156,7 @@ def main():
                 name="Cmake configuration",
                 command=cmake_cmd,
                 workdir=build_dir,
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -176,6 +177,7 @@ def main():
                 name="Build ClickHouse",
                 command=f"ninja {targets}",
                 workdir=build_dir,
+                with_log=True,
             )
         )
         run_shell("sccache stats", "sccache --show-stats")
@@ -206,6 +208,7 @@ def main():
                     f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={temp_dir} BUILD_TYPE={BUILD_TYPE_TO_DEB_PACKAGE_TYPE[build_type]} VERSION_STRING={version_dict['string']} DEB_ARCH={deb_arch} ./build --deb {'--rpm --tgz' if 'release' in build_type else ''}",
                 ],
                 workdir=build_dir,
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/clickbench.py b/ci/jobs/clickbench.py
@@ -36,6 +36,7 @@ def start():
             Result.from_commands_run(
                 name="Start ClickHouse",
                 command=[start, log_export_config, setup_logs_replication],
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/collect_statistics.py b/ci/jobs/collect_statistics.py
@@ -139,7 +139,9 @@ def get_all_job_names():
         return True
 
     results.append(
-        Result.from_commands_run(name="Get all job names", command=get_all_job_names)
+        Result.from_commands_run(
+            name="Get all job names", command=get_all_job_names, with_info=True
+        )
     )
     if not results[-1].is_ok():
         sys.exit()
diff --git a/ci/jobs/docker_server_job.py b/ci/jobs/docker_server_job.py
@@ -103,7 +103,11 @@ def main():
     command = f"docker buildx build --platform {platforms} \
                     -t {image_repo}:tmp {image_path} -f {docker_path}/Dockerfile.ubuntu"
 
-    results.append(Result.from_commands_run(name=f"{image_repo}:tmp", command=command))
+    results.append(
+        Result.from_commands_run(
+            name=f"{image_repo}:tmp", command=command, with_log=True
+        )
+    )
 
     Result.create_from(results=results, stopwatch=stopwatch).complete_job()
 
diff --git a/ci/jobs/fast_test.py b/ci/jobs/fast_test.py
@@ -175,6 +175,7 @@ def main():
                 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 -DENABLE_NURAFT=1 -DENABLE_SIMDJSON=1 \
                 -DENABLE_JEMALLOC=1 -DENABLE_LIBURING=1 -DENABLE_YAML_CPP=1 -DCOMPILER_CACHE=sccache",
                 workdir=build_dir,
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -186,6 +187,7 @@ def main():
                 name="Build ClickHouse",
                 command="ninja clickhouse-bundle clickhouse-stripped",
                 workdir=build_dir,
+                with_log=True,
             )
         )
         Shell.check("sccache --show-stats")
@@ -203,6 +205,7 @@ def main():
                 name="Check and Compress binary",
                 command=commands,
                 workdir=build_dir,
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -220,6 +223,7 @@ def main():
             Result.from_commands_run(
                 name="Install ClickHouse Config",
                 command=commands,
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/functional_stateful_tests.py b/ci/jobs/functional_stateful_tests.py
@@ -104,7 +104,9 @@ def main():
             f"clickhouse-server --version",
         ]
         results.append(
-            Result.from_commands_run(name="Install ClickHouse", command=commands)
+            Result.from_commands_run(
+                name="Install ClickHouse", command=commands, with_log=True
+            )
         )
         res = results[-1].is_ok()
 
diff --git a/ci/jobs/functional_stateless_tests.py b/ci/jobs/functional_stateless_tests.py
@@ -121,7 +121,9 @@ def main():
             f"clickhouse-server --version",
         ]
         results.append(
-            Result.from_commands_run(name="Install ClickHouse", command=commands)
+            Result.from_commands_run(
+                name="Install ClickHouse", command=commands, with_log=True
+            )
         )
         res = results[-1].is_ok()
 
diff --git a/ci/jobs/fuzzers_job.py b/ci/jobs/fuzzers_job.py
@@ -38,6 +38,7 @@ def start():
             Result.from_commands_run(
                 name="Start ClickHouse",
                 command=[start, log_export_config, setup_logs_replication],
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
@@ -50,6 +51,7 @@ def start():
             Result.from_commands_run(
                 name="Start ClickHouse",
                 command=commands,
+                with_log=True,
             )
         )
         res = results[-1].is_ok()
diff --git a/ci/jobs/performance_tests.py b/ci/jobs/performance_tests.py
@@ -331,7 +331,9 @@ def main():
             "clickhouse-local --version",
         ]
         results.append(
-            Result.from_commands_run(name="Install ClickHouse", command=commands)
+            Result.from_commands_run(
+                name="Install ClickHouse", command=commands, with_log=True
+            )
         )
         res = results[-1].is_ok()
 
@@ -350,7 +352,7 @@ def main():
             ]
             results.append(
                 Result.from_commands_run(
-                    name="Install Reference ClickHouse", command=commands
+                    name="Install Reference ClickHouse", command=commands, with_log=True
                 )
             )
             res = results[-1].is_ok()
@@ -419,7 +421,9 @@ def restart_ch():
             f"cp -R {temp_dir}/coordination0 {perf_left}/coordination",
             f"cp -R {temp_dir}/coordination0 {perf_right}/coordination",
         ]
-        results.append(Result.from_commands_run(name="Configure", command=commands))
+        results.append(
+            Result.from_commands_run(name="Configure", command=commands, with_log=True)
+        )
         res = results[-1].is_ok()
 
     leftCH = CHServer(is_left=True)
@@ -520,6 +524,7 @@ def run_tests():
             Result.from_commands_run(
                 name="Report",
                 command=commands,
+                with_log=True,
                 workdir=perf_wd,
             )
         )
diff --git a/ci/praktika/docker.py b/ci/praktika/docker.py
@@ -59,7 +59,9 @@ def build(cls, config: "Docker.Config", digests, amd_only, arm_only, with_log):
 
             command = f"docker buildx build --builder default {tags_substr} {from_tag} --platform {','.join(platforms)} --cache-to type=inline --cache-from type=registry,ref={config.name} {config.path} --push"
 
-            return Result.from_commands_run(name=name, command=command)
+            return Result.from_commands_run(
+                name=name, command=command, with_info=with_log
+            )
         else:
             return Result(
                 name=name,
@@ -103,6 +105,7 @@ def merge_manifest(
         return Result.from_commands_run(
             name=f"merge: {config.name}:{digests[config.name]} (latest={add_latest})",
             command=commands,
+            with_info=with_log,
             fail_fast=True,
         )
 
diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py
@@ -186,6 +186,7 @@ def _check_yaml_up_to_date():
         return Result.from_commands_run(
             name="Check Workflows",
             command=commands,
+            with_info=True,
             fail_fast=True,
         )
 
@@ -268,7 +269,9 @@ def _check_db(workflow):
                 name = pre_check.__name__
             else:
                 name = str(pre_check)
-            res_.append(Result.from_commands_run(name=name, command=pre_check))
+            res_.append(
+                Result.from_commands_run(name=name, command=pre_check, with_info=True)
+            )
 
         results.append(
             Result.create_from(name="Pre Hooks", results=res_, stopwatch=sw_)
@@ -425,6 +428,7 @@ def check_affected_jobs():
             Result.from_commands_run(
                 name="Filter not affected jobs",
                 command=check_affected_jobs,
+                with_info=True,
             )
         )
 
@@ -493,7 +497,9 @@ def _finish_workflow(workflow, job_name):
                 name = check.__name__
             else:
                 name = str(check)
-            results_.append(Result.from_commands_run(name=name, command=check))
+            results_.append(
+                Result.from_commands_run(name=name, command=check, with_info=True)
+            )
 
         results.append(
             Result.create_from(name="Post Hooks", results=results_, stopwatch=sw_)
diff --git a/ci/praktika/result.py b/ci/praktika/result.py
@@ -386,7 +386,6 @@ def from_commands_run(
         command,
         with_log=False,
         with_info=False,
-        with_info_on_failure=True,
         fail_fast=True,
         workdir=None,
         command_args=None,
@@ -400,7 +399,6 @@ def from_commands_run(
         :param workdir: Optional working directory.
         :param with_log: Boolean flag to log output to a file.
         :param with_info: Fill in Result.info from command output
-        :param with_info_on_failure: Fill in Result.info from command output on failure only
         :param fail_fast: Boolean flag to stop execution if one command fails.
         :param command_args: Positional arguments for the callable command.
         :param command_kwargs: Keyword arguments for the callable command.
@@ -415,7 +413,7 @@ def from_commands_run(
         # Set log file path if logging is enabled
         if with_log:
             log_file = f"{Utils.absolute_path(Settings.TEMP_DIR)}/{Utils.normalize_string(name)}.log"
-        elif with_info or with_info_on_failure:
+        elif with_info:
             log_file = f"/tmp/praktika_{Utils.normalize_string(name)}.log"
         else:
             log_file = None
@@ -432,23 +430,22 @@ def from_commands_run(
             for command_ in command:
                 if callable(command_):
                     # If command is a Python function, call it with provided arguments
-                    if with_info or with_info_on_failure:
+                    if with_info:
                         buffer = io.StringIO()
                         with redirect_stdout(buffer):
                             result = command_(*command_args, **command_kwargs)
+                        error_infos = buffer.getvalue()
                     else:
                         result = command_(*command_args, **command_kwargs)
                     if isinstance(result, bool):
                         res = result
                     elif result:
                         error_infos.append(str(result))
                         res = False
-                    if with_info or (with_info_on_failure and not res):
-                        error_infos = buffer.getvalue()
                 else:
                     # Run shell command in a specified directory with logging and verbosity
                     exit_code = Shell.run(command_, verbose=True, log_file=log_file)
-                    if with_info or (with_info_on_failure and exit_code != 0):
+                    if with_info:
                         with open(log_file, "r") as f:
                             error_infos.append(f.read().strip())
                     res = exit_code == 0
@@ -459,17 +456,11 @@ def from_commands_run(
                     break
 
         # Create and return the result object with status and log file (if any)
-        MAX_LINES_IN_INFO = 100
         return Result.create_from(
             name=name,
             status=res,
             stopwatch=stop_watch_,
-            info=(
-                error_infos
-                if len(error_infos) < MAX_LINES_IN_INFO
-                else [f" ~~~ truncated {len(error_infos)-MAX_LINES_IN_INFO} lines ~~~"]
-                + error_infos[-MAX_LINES_IN_INFO:]
-            ),
+            info=error_infos,
             files=[log_file] if with_log else None,
         )
 
diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py
@@ -373,7 +373,9 @@ def _post_run(
                     name = check.__name__
                 else:
                     name = str(check)
-                results_.append(Result.from_commands_run(name=name, command=check))
+                results_.append(
+                    Result.from_commands_run(name=name, command=check, with_info=True)
+                )
             result.results.append(
                 Result.create_from(name="Post Hooks", results=results_, stopwatch=sw_)
             )

Original file line number	Diff line number	Diff line change
`@@ -156,6 +156,7 @@ def main():`
`156`	`156`	`name="Cmake configuration",`
`157`	`157`	`command=cmake_cmd,`
`158`	`158`	`workdir=build_dir,`
	`159`	`+ with_log=True,`
`159`	`160`	`)`
`160`	`161`	`)`
`161`	`162`	`res = results[-1].is_ok()`
`@@ -176,6 +177,7 @@ def main():`
`176`	`177`	`name="Build ClickHouse",`
`177`	`178`	`command=f"ninja {targets}",`
`178`	`179`	`workdir=build_dir,`
	`180`	`+ with_log=True,`
`179`	`181`	`)`
`180`	`182`	`)`
`181`	`183`	`run_shell("sccache stats", "sccache --show-stats")`
`@@ -206,6 +208,7 @@ def main():`
`206`	`208`	`f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={temp_dir} BUILD_TYPE={BUILD_TYPE_TO_DEB_PACKAGE_TYPE[build_type]} VERSION_STRING={version_dict['string']} DEB_ARCH={deb_arch} ./build --deb {'--rpm --tgz' if 'release' in build_type else ''}",`
`207`	`209`	`],`
`208`	`210`	`workdir=build_dir,`
	`211`	`+ with_log=True,`
`209`	`212`	`)`
`210`	`213`	`)`
`211`	`214`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,7 @@ def start():`
`36`	`36`	`Result.from_commands_run(`
`37`	`37`	`name="Start ClickHouse",`
`38`	`38`	`command=[start, log_export_config, setup_logs_replication],`
	`39`	`+ with_log=True,`
`39`	`40`	`)`
`40`	`41`	`)`
`41`	`42`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -175,6 +175,7 @@ def main():`
`175`	`175`	`-DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 -DENABLE_NURAFT=1 -DENABLE_SIMDJSON=1 \`
`176`	`176`	`-DENABLE_JEMALLOC=1 -DENABLE_LIBURING=1 -DENABLE_YAML_CPP=1 -DCOMPILER_CACHE=sccache",`
`177`	`177`	`workdir=build_dir,`
	`178`	`+ with_log=True,`
`178`	`179`	`)`
`179`	`180`	`)`
`180`	`181`	`res = results[-1].is_ok()`
`@@ -186,6 +187,7 @@ def main():`
`186`	`187`	`name="Build ClickHouse",`
`187`	`188`	`command="ninja clickhouse-bundle clickhouse-stripped",`
`188`	`189`	`workdir=build_dir,`
	`190`	`+ with_log=True,`
`189`	`191`	`)`
`190`	`192`	`)`
`191`	`193`	`Shell.check("sccache --show-stats")`
`@@ -203,6 +205,7 @@ def main():`
`203`	`205`	`name="Check and Compress binary",`
`204`	`206`	`command=commands,`
`205`	`207`	`workdir=build_dir,`
	`208`	`+ with_log=True,`
`206`	`209`	`)`
`207`	`210`	`)`
`208`	`211`	`res = results[-1].is_ok()`
`@@ -220,6 +223,7 @@ def main():`
`220`	`223`	`Result.from_commands_run(`
`221`	`224`	`name="Install ClickHouse Config",`
`222`	`225`	`command=commands,`
	`226`	`+ with_log=True,`
`223`	`227`	`)`
`224`	`228`	`)`
`225`	`229`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -104,7 +104,9 @@ def main():`
`104`	`104`	`f"clickhouse-server --version",`
`105`	`105`	`]`
`106`	`106`	`results.append(`
`107`		`- Result.from_commands_run(name="Install ClickHouse", command=commands)`
	`107`	`+ Result.from_commands_run(`
	`108`	`+ name="Install ClickHouse", command=commands, with_log=True`
	`109`	`+ )`
`108`	`110`	`)`
`109`	`111`	`res = results[-1].is_ok()`
`110`	`112`
Original file line number	Diff line number	Diff line change
`@@ -121,7 +121,9 @@ def main():`
`121`	`121`	`f"clickhouse-server --version",`
`122`	`122`	`]`
`123`	`123`	`results.append(`
`124`		`- Result.from_commands_run(name="Install ClickHouse", command=commands)`
	`124`	`+ Result.from_commands_run(`
	`125`	`+ name="Install ClickHouse", command=commands, with_log=True`
	`126`	`+ )`
`125`	`127`	`)`
`126`	`128`	`res = results[-1].is_ok()`
`127`	`129`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ def start():`
`38`	`38`	`Result.from_commands_run(`
`39`	`39`	`name="Start ClickHouse",`
`40`	`40`	`command=[start, log_export_config, setup_logs_replication],`
	`41`	`+ with_log=True,`
`41`	`42`	`)`
`42`	`43`	`)`
`43`	`44`	`res = results[-1].is_ok()`
`@@ -50,6 +51,7 @@ def start():`
`50`	`51`	`Result.from_commands_run(`
`51`	`52`	`name="Start ClickHouse",`
`52`	`53`	`command=commands,`
	`54`	`+ with_log=True,`
`53`	`55`	`)`
`54`	`56`	`)`
`55`	`57`	`res = results[-1].is_ok()`
Original file line number	Diff line number	Diff line change
`@@ -331,7 +331,9 @@ def main():`
`331`	`331`	`"clickhouse-local --version",`
`332`	`332`	`]`
`333`	`333`	`results.append(`
`334`		`- Result.from_commands_run(name="Install ClickHouse", command=commands)`
	`334`	`+ Result.from_commands_run(`
	`335`	`+ name="Install ClickHouse", command=commands, with_log=True`
	`336`	`+ )`
`335`	`337`	`)`
`336`	`338`	`res = results[-1].is_ok()`
`337`	`339`
`@@ -350,7 +352,7 @@ def main():`
`350`	`352`	`]`
`351`	`353`	`results.append(`
`352`	`354`	`Result.from_commands_run(`
`353`		`- name="Install Reference ClickHouse", command=commands`
	`355`	`+ name="Install Reference ClickHouse", command=commands, with_log=True`
`354`	`356`	`)`
`355`	`357`	`)`
`356`	`358`	`res = results[-1].is_ok()`
`@@ -419,7 +421,9 @@ def restart_ch():`
`419`	`421`	`f"cp -R {temp_dir}/coordination0 {perf_left}/coordination",`
`420`	`422`	`f"cp -R {temp_dir}/coordination0 {perf_right}/coordination",`
`421`	`423`	`]`
`422`		`- results.append(Result.from_commands_run(name="Configure", command=commands))`
	`424`	`+ results.append(`
	`425`	`+ Result.from_commands_run(name="Configure", command=commands, with_log=True)`
	`426`	`+ )`
`423`	`427`	`res = results[-1].is_ok()`
`424`	`428`
`425`	`429`	`leftCH = CHServer(is_left=True)`
`@@ -520,6 +524,7 @@ def run_tests():`
`520`	`524`	`Result.from_commands_run(`
`521`	`525`	`name="Report",`
`522`	`526`	`command=commands,`
	`527`	`+ with_log=True,`
`523`	`528`	`workdir=perf_wd,`
`524`	`529`	`)`
`525`	`530`	`)`