Skip to content

[YSQL] Parallelize postgres build make invocations #28542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[YSQL] Parallelize postgres build make invocations #28542

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 81 additions & 43 deletions python/yugabyte/build_postgres.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

from contextlib import contextmanager
from overrides import overrides
Expand Down Expand Up @@ -810,8 +811,6 @@ def make_postgres(self) -> None:

env_script_content = self.get_env_script_content()

pg_compile_commands_paths = []

external_extension_dirs = [os.path.join(self.pg_build_root, d) for d
in ('third-party-extensions', 'yb-extensions')]
work_dirs = [
Expand All @@ -821,47 +820,36 @@ def make_postgres(self) -> None:
os.path.join(self.pg_build_root, 'src/tools/pg_bsd_indent'),
] + external_extension_dirs

# TODO(#27196): parallelize this for loop.
for work_dir in work_dirs:
# Postgresql requires MAKELEVEL to be 0 or non-set when calling its make.
# But in the case where the YB project is built with make,
# MAKELEVEL is not 0 at this point. We temporarily unset MAKELEVEL to
# deal with this.
with WorkDirContext(work_dir), SavedEnviron('MAKELEVEL'):
self.write_debug_scripts(env_script_content)

make_cmd_suffix = []
if work_dir in external_extension_dirs:
make_cmd_suffix = ['PG_CONFIG=' + self.pg_config_path]

# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)

complete_make_cmd = make_cmd + make_cmd_suffix
complete_make_cmd_str = shlex_join(complete_make_cmd)
self.run_make_with_retries(work_dir, complete_make_cmd_str)

if self.build_type != 'compilecmds' or work_dir == self.pg_build_root:
self.run_make_install(make_cmd, make_cmd_suffix)
else:
logging.info(
"Not running 'make install' in the %s directory since we are only "
"generating the compilation database", work_dir)

if self.export_compile_commands and not self.skip_pg_compile_commands:
logging.info("Generating the compilation database in directory '%s'", work_dir)

compile_commands_path = os.path.join(work_dir, 'compile_commands.json')
with SavedEnviron(YB_PG_SKIP_CONFIG_STATUS='1'):
if not os.path.exists(compile_commands_path):
run_program(
['compiledb', 'make', '-n'] + make_cmd_suffix, capture_output=False)

if not os.path.exists(compile_commands_path):
raise RuntimeError("Failed to generate compilation database at: %s" %
compile_commands_path)
pg_compile_commands_paths.append(compile_commands_path)
# Parallelize the build process for work_dirs
# Use ThreadPoolExecutor to run builds in parallel
max_workers = min(len(work_dirs), multiprocessing.cpu_count())
logging.info("Starting parallel build with %d workers for %d work directories",
max_workers, len(work_dirs))

pg_compile_commands_paths = []
start_time_sec = time.time()

with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all work directory builds to the executor
future_to_work_dir = {
executor.submit(self.build_work_dir, work_dir, make_cmd, env_script_content,
external_extension_dirs): work_dir
for work_dir in work_dirs
}

# Collect results as they complete
for future in as_completed(future_to_work_dir):
work_dir = future_to_work_dir[future]
try:
compile_commands_paths = future.result()
pg_compile_commands_paths.extend(compile_commands_paths)
logging.info("Completed build for work directory: %s", work_dir)
except Exception as exc:
logging.error("Build failed for work directory %s: %s", work_dir, exc)
raise RuntimeError(f"PostgreSQL build failed in directory {work_dir}: {exc}")

elapsed_time_sec = time.time() - start_time_sec
logging.info("Parallel build completed in %.2f sec", elapsed_time_sec)

if self.export_compile_commands:
self.write_compile_commands_files(pg_compile_commands_paths)
Expand Down Expand Up @@ -911,6 +899,56 @@ def run_make_with_retries(self, work_dir: str, complete_make_cmd_str: str) -> No
raise RuntimeError(
f"Maximum build attempts reached ({TRANSIENT_BUILD_RETRIES} attempts).")

def build_work_dir(self, work_dir: str, make_cmd: List[str], env_script_content: str,
external_extension_dirs: List[str]) -> List[str]:
"""
Build a single work directory. Returns list of compile commands paths.
This method is designed to be called in parallel.
"""
pg_compile_commands_paths = []

# Postgresql requires MAKELEVEL to be 0 or non-set when calling its make.
# But in the case where the YB project is built with make,
# MAKELEVEL is not 0 at this point. We temporarily unset MAKELEVEL to
# deal with this.
with WorkDirContext(work_dir), SavedEnviron('MAKELEVEL'):
self.write_debug_scripts(env_script_content)

make_cmd_suffix = []
if work_dir in external_extension_dirs:
make_cmd_suffix = ['PG_CONFIG=' + self.pg_config_path]

# Actually run Make.
if is_verbose_mode():
logging.info("Running make in the %s directory", work_dir)

complete_make_cmd = make_cmd + make_cmd_suffix
complete_make_cmd_str = shlex_join(complete_make_cmd)
self.run_make_with_retries(work_dir, complete_make_cmd_str)

if self.build_type != 'compilecmds' or work_dir == self.pg_build_root:
self.run_make_install(make_cmd, make_cmd_suffix)
else:
logging.info(
"Not running 'make install' in the %s directory since we are only "
"generating the compilation database", work_dir)

if self.export_compile_commands and not self.skip_pg_compile_commands:
logging.info("Generating the compilation database in directory '%s'", work_dir)

compile_commands_path = os.path.join(work_dir, 'compile_commands.json')
with SavedEnviron(YB_PG_SKIP_CONFIG_STATUS='1'):
if not os.path.exists(compile_commands_path):
run_program(
['compiledb', 'make', '-n'] + make_cmd_suffix, capture_output=False)

if not os.path.exists(compile_commands_path):
raise RuntimeError("Failed to generate compilation database at: %s" %
compile_commands_path)
pg_compile_commands_paths.append(compile_commands_path)

return pg_compile_commands_paths

def get_env_script_content(self) -> str:
"""
Returns a Bash script that sets all variables necessary to easily rerun the "make" step
Expand Down