From 61483134a7f938b5c70118f2c86f8377ec1fc373 Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Fri, 1 Nov 2024 08:01:18 +0100 Subject: [PATCH 1/7] add arg for custom toolchain * add utils for duplicate methods --- build-system/cpp-cmake-conan/src/build.py | 72 +++++------ .../cpp-cmake-conan/src/install_deps.py | 114 +++++++++++------- build-system/cpp-cmake-conan/src/utils.py | 63 ++++++++++ 3 files changed, 164 insertions(+), 85 deletions(-) create mode 100644 build-system/cpp-cmake-conan/src/utils.py diff --git a/build-system/cpp-cmake-conan/src/build.py b/build-system/cpp-cmake-conan/src/build.py index 0caafab3..9fb1bdcd 100644 --- a/build-system/cpp-cmake-conan/src/build.py +++ b/build-system/cpp-cmake-conan/src/build.py @@ -12,45 +12,23 @@ # # SPDX-License-Identifier: Apache-2.0 -import json import os import subprocess from argparse import ArgumentParser from pathlib import Path -from typing import List -from velocitas_lib import get_valid_arch, get_workspace_dir +from utils import ( + get_build_folder, + get_build_tools_path, + load_toolchain, + safe_get_workspace_dir, +) +from velocitas_lib import get_valid_arch CMAKE_EXECUTABLE = "cmake" CONAN_EXECUTABLE = "conan" -def safe_get_workspace_dir() -> str: - """A safe version of get_workspace_dir which defaults to '.'.""" - try: - return get_workspace_dir() - except Exception: - return os.path.abspath(".") - - -def get_build_folder(build_arch: str, host_arch: str): - if host_arch == build_arch: - return os.path.join(safe_get_workspace_dir(), "build") - return os.path.join(safe_get_workspace_dir(), f"build_linux_{host_arch}") - - -def get_build_tools_path(build_folder_path: str) -> str: - paths: List[str] = [] - with open( - os.path.join(build_folder_path, "conanbuildinfo.txt"), encoding="utf-8" - ) as file: - for line in file: - if line.startswith("PATH="): - path_list = json.loads(line[len("PATH=") :]) - paths.extend(path_list) - return ";".join(paths) - - def print_build_info( build_variant: str, build_arch: str, @@ -89,22 +67,30 @@ def build( build_target: str, static_build: bool, coverage: bool = True, + toolchain_file: str = "", ) -> None: - cxx_flags = ["-g"] - if coverage: - cxx_flags.append("--coverage") - - if build_variant == "release": - cxx_flags.append("-s") - cxx_flags.append("-O3") + xcompile_toolchain_file = "" + if toolchain_file != "": + load_toolchain(toolchain_file) + xcompile_toolchain_file = f"-DCMAKE_TOOLCHAIN_FILE={os.path.join(safe_get_workspace_dir(),'OEToolchainConfig.cmake')}" + cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={os.environ.get('CXXFLAGS', '')}" else: - cxx_flags.append("-O0") + cxx_flags = ["-g"] + if coverage: + cxx_flags.append("--coverage") + + if build_variant == "release": + cxx_flags.append("-s") + cxx_flags.append("-O3") + else: + cxx_flags.append("-O0") + + cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={' '.join(cxx_flags)}" build_folder = get_build_folder(build_arch, host_arch) os.makedirs(build_folder, exist_ok=True) - xcompile_toolchain_file = "" - if build_arch != host_arch: + if build_arch != host_arch and xcompile_toolchain_file == "": profile_build_path = ( Path(__file__) .absolute() @@ -126,7 +112,7 @@ def build( "-B.", "-G", "Ninja", - f"-DCMAKE_CXX_FLAGS={' '.join(cxx_flags)}", + cmake_cxx_flags, ], cwd=build_folder, ) @@ -172,6 +158,10 @@ def cli() -> None: parser.add_argument( "-s", "--static", action="store_true", help="Links all dependencies statically." ) + parser.add_argument( + "--toolchain", + help="Specify a file (absolute path) containing the definitions of a custom toolchain.", + ) parser.add_argument( "-x", "--cross", @@ -193,7 +183,7 @@ def cli() -> None: host_arch = get_valid_arch(host_arch) print_build_info(args.variant, build_arch, host_arch, args.target, args.static) - build(args.variant, build_arch, host_arch, args.target, args.static) + build(args.variant, build_arch, host_arch, args.target, args.static, args.toolchain) if __name__ == "__main__": diff --git a/build-system/cpp-cmake-conan/src/install_deps.py b/build-system/cpp-cmake-conan/src/install_deps.py index 7b2caa58..29065471 100644 --- a/build-system/cpp-cmake-conan/src/install_deps.py +++ b/build-system/cpp-cmake-conan/src/install_deps.py @@ -22,21 +22,8 @@ from argparse import ArgumentParser from pathlib import Path -from velocitas_lib import get_valid_arch, get_workspace_dir - - -def safe_get_workspace_dir() -> str: - """A safe version of get_workspace_dir which defaults to '.'.""" - try: - return get_workspace_dir() - except Exception: - return os.path.abspath(".") - - -def get_build_folder(build_arch: str, host_arch: str): - if host_arch == build_arch: - return os.path.join(safe_get_workspace_dir(), "build") - return os.path.join(safe_get_workspace_dir(), f"build_linux_{host_arch}") +from utils import get_build_folder, load_toolchain, safe_get_workspace_dir +from velocitas_lib import get_valid_arch def get_profile_name(arch: str, build_variant: str) -> str: @@ -53,11 +40,31 @@ def get_profile_name(arch: str, build_variant: str) -> str: return f"linux_{get_valid_arch(arch)}_{build_variant}" +def get_valid_conan_arch(arch: str) -> str: + """Return the valid architecture for the given `arch`. + + Args: + arch (str): The architecture to validate. + + Returns: + str: The valid architecture. + """ + if arch == "x86_64": + return "x86_64" + elif arch == "aarch64" or arch == "armv8" or arch == "arm64": + return "armv8" + elif arch == "armv7": + return "armv7" + else: + raise ValueError(f"Unsupported architecture: {arch}") + + def install_deps_via_conan( build_arch: str, host_arch: str, is_debug: bool = False, build_all_deps: bool = False, + toolchain_file: str = "", ) -> None: build_variant = "debug" if is_debug else "release" @@ -68,42 +75,52 @@ def install_deps_via_conan( ".conan", "profiles", get_profile_name(build_arch, build_variant) ) ) - - profile_host_path = ( - Path(__file__) - .absolute() - .parent.joinpath( - ".conan", "profiles", get_profile_name(host_arch, build_variant) + if toolchain_file != "": + load_toolchain(toolchain_file) + host_config = [ + "-s:h", + f"arch={get_valid_conan_arch(os.environ.get('OECORE_TARGET_ARCH','').strip())}", + "-s:h", + f"arch_build={get_valid_conan_arch(build_arch)}", + ] + else: + profile_host_path = ( + Path(__file__) + .absolute() + .parent.joinpath( + ".conan", "profiles", get_profile_name(host_arch, build_variant) + ) ) - ) + host_config = ["-pr:h", f"{profile_host_path}"] build_folder = get_build_folder(build_arch, host_arch) os.makedirs(build_folder, exist_ok=True) deps_to_build = "missing" if not build_all_deps else "*" - toolchain = f"/usr/bin/{host_arch}-linux-gnu" - build_host = f"{host_arch}-linux-gnu" - cc_compiler = "gcc" - cxx_compiler = "g++" - - os.environ["CONAN_CMAKE_FIND_ROOT_PATH"] = toolchain - os.environ["CONAN_CMAKE_SYSROOT"] = toolchain - os.environ["CC"] = f"{build_host}-{cc_compiler}" - os.environ["CXX"] = f"{build_host}-{cxx_compiler}" - + if toolchain_file == "": + toolchain = f"/usr/bin/{host_arch}-linux-gnu" + build_host = f"{host_arch}-linux-gnu" + cc_compiler = "gcc" + cxx_compiler = "g++" + + os.environ["CONAN_CMAKE_FIND_ROOT_PATH"] = toolchain + os.environ["CONAN_CMAKE_SYSROOT"] = toolchain + os.environ["CC"] = f"{build_host}-{cc_compiler}" + os.environ["CXX"] = f"{build_host}-{cxx_compiler}" + + args = [ + "conan", + "install", + *host_config, + "-pr:b", + str(profile_build_path), + "--build", + deps_to_build, + safe_get_workspace_dir(), + ] subprocess.check_call( - [ - "conan", - "install", - "-pr:h", - profile_host_path, - "-pr:b", - profile_build_path, - "--build", - deps_to_build, - safe_get_workspace_dir(), - ], + args, env=os.environ, cwd=build_folder, ) @@ -129,6 +146,11 @@ def cli() -> None: action="store_true", help="Forces all dependencies to be rebuild from source.", ) + argument_parser.add_argument( + "--toolchain", + default="", + help="Specify a file (absolute path) containing the definitions of a custom toolchain.", + ) argument_parser.add_argument( "-x", "--cross", @@ -149,7 +171,11 @@ def cli() -> None: subprocess.check_call(["conan", "config", "set", "general.revisions_enabled=1"]) install_deps_via_conan( - build_arch, host_arch, args.debug and not args.release, args.build_all_deps + build_arch, + host_arch, + args.debug and not args.release, + args.build_all_deps, + args.toolchain, ) diff --git a/build-system/cpp-cmake-conan/src/utils.py b/build-system/cpp-cmake-conan/src/utils.py new file mode 100644 index 00000000..2c9edd50 --- /dev/null +++ b/build-system/cpp-cmake-conan/src/utils.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import subprocess +from typing import List + +from velocitas_lib import get_workspace_dir + + +def safe_get_workspace_dir() -> str: + """A safe version of get_workspace_dir which defaults to '.'.""" + try: + return get_workspace_dir().strip() + except Exception: + return os.path.abspath(".") + + +def get_build_folder(build_arch: str, host_arch: str): + if host_arch == build_arch: + return os.path.join(safe_get_workspace_dir(), "build") + return os.path.join(safe_get_workspace_dir(), f"build_linux_{host_arch}") + + +def get_build_tools_path(build_folder_path: str) -> str: + paths: List[str] = [] + with open( + os.path.join(build_folder_path, "conanbuildinfo.txt"), encoding="utf-8" + ) as file: + for line in file: + if line.startswith("PATH="): + path_list = json.loads(line[len("PATH=") :]) + paths.extend(path_list) + return ";".join(paths) + + +def load_toolchain(toolchain_file: str) -> None: + if not os.path.exists(toolchain_file): + raise FileNotFoundError(f"Toolchain file {toolchain_file} not found.") + print(f"Loading toolchain file {toolchain_file}") + + proc = subprocess.Popen( + ["env", "-i", "bash", "-c", f"source {toolchain_file} && env"], + stdout=subprocess.PIPE, + ) + if proc.stdout is not None: + for line in proc.stdout: + (key, _, value) = line.decode().partition("=") + print(f"Setting {key} to {value}") + os.environ[key] = value + proc.communicate() From c910de2e114e5f5599c0cd454d82837c4d897d0c Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Fri, 1 Nov 2024 10:38:03 +0100 Subject: [PATCH 2/7] fix indentation --- build-system/cpp-cmake-conan/src/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/cpp-cmake-conan/src/build.py b/build-system/cpp-cmake-conan/src/build.py index 9fb1bdcd..57bc057a 100644 --- a/build-system/cpp-cmake-conan/src/build.py +++ b/build-system/cpp-cmake-conan/src/build.py @@ -85,7 +85,7 @@ def build( else: cxx_flags.append("-O0") - cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={' '.join(cxx_flags)}" + cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={' '.join(cxx_flags)}" build_folder = get_build_folder(build_arch, host_arch) os.makedirs(build_folder, exist_ok=True) From e19d4b2e556e693a5568c4ac09500e0a23388499 Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Fri, 1 Nov 2024 11:25:52 +0100 Subject: [PATCH 3/7] fix default --- build-system/cpp-cmake-conan/src/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-system/cpp-cmake-conan/src/build.py b/build-system/cpp-cmake-conan/src/build.py index 57bc057a..96f10b17 100644 --- a/build-system/cpp-cmake-conan/src/build.py +++ b/build-system/cpp-cmake-conan/src/build.py @@ -66,8 +66,8 @@ def build( host_arch: str, build_target: str, static_build: bool, - coverage: bool = True, toolchain_file: str = "", + coverage: bool = True, ) -> None: xcompile_toolchain_file = "" if toolchain_file != "": @@ -160,6 +160,7 @@ def cli() -> None: ) parser.add_argument( "--toolchain", + default="", help="Specify a file (absolute path) containing the definitions of a custom toolchain.", ) parser.add_argument( From 74ea437e26c7428f66ed421961a6a90154ac047a Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Tue, 5 Nov 2024 13:23:26 +0100 Subject: [PATCH 4/7] fix host_arch setting * correct name for build folder --- build-system/cpp-cmake-conan/src/build.py | 3 +++ build-system/cpp-cmake-conan/src/install_deps.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/build-system/cpp-cmake-conan/src/build.py b/build-system/cpp-cmake-conan/src/build.py index 96f10b17..84691be5 100644 --- a/build-system/cpp-cmake-conan/src/build.py +++ b/build-system/cpp-cmake-conan/src/build.py @@ -74,6 +74,9 @@ def build( load_toolchain(toolchain_file) xcompile_toolchain_file = f"-DCMAKE_TOOLCHAIN_FILE={os.path.join(safe_get_workspace_dir(),'OEToolchainConfig.cmake')}" cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={os.environ.get('CXXFLAGS', '')}" + host_arch = get_valid_arch( + os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() + ) else: cxx_flags = ["-g"] if coverage: diff --git a/build-system/cpp-cmake-conan/src/install_deps.py b/build-system/cpp-cmake-conan/src/install_deps.py index 29065471..39b5bea0 100644 --- a/build-system/cpp-cmake-conan/src/install_deps.py +++ b/build-system/cpp-cmake-conan/src/install_deps.py @@ -77,9 +77,12 @@ def install_deps_via_conan( ) if toolchain_file != "": load_toolchain(toolchain_file) + host_arch = get_valid_arch( + os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() + ) host_config = [ "-s:h", - f"arch={get_valid_conan_arch(os.environ.get('OECORE_TARGET_ARCH','').strip())}", + f"arch={get_valid_conan_arch(host_arch)}", "-s:h", f"arch_build={get_valid_conan_arch(build_arch)}", ] From 84c729176ce71b849c565ae52757ee0b26733c1d Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Tue, 5 Nov 2024 13:42:17 +0100 Subject: [PATCH 5/7] fix arch --- build-system/cpp-cmake-conan/src/build.py | 5 ++--- build-system/cpp-cmake-conan/src/install_deps.py | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/build-system/cpp-cmake-conan/src/build.py b/build-system/cpp-cmake-conan/src/build.py index 84691be5..e0b6dafd 100644 --- a/build-system/cpp-cmake-conan/src/build.py +++ b/build-system/cpp-cmake-conan/src/build.py @@ -74,9 +74,8 @@ def build( load_toolchain(toolchain_file) xcompile_toolchain_file = f"-DCMAKE_TOOLCHAIN_FILE={os.path.join(safe_get_workspace_dir(),'OEToolchainConfig.cmake')}" cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={os.environ.get('CXXFLAGS', '')}" - host_arch = get_valid_arch( - os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() - ) + host_arch = os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() + else: cxx_flags = ["-g"] if coverage: diff --git a/build-system/cpp-cmake-conan/src/install_deps.py b/build-system/cpp-cmake-conan/src/install_deps.py index 39b5bea0..5e907ef2 100644 --- a/build-system/cpp-cmake-conan/src/install_deps.py +++ b/build-system/cpp-cmake-conan/src/install_deps.py @@ -49,11 +49,11 @@ def get_valid_conan_arch(arch: str) -> str: Returns: str: The valid architecture. """ - if arch == "x86_64": + if arch == "x86_64" or arch == "amd64": return "x86_64" elif arch == "aarch64" or arch == "armv8" or arch == "arm64": return "armv8" - elif arch == "armv7": + elif arch == "armv7" or arch == "arm" or arch == "arm32": return "armv7" else: raise ValueError(f"Unsupported architecture: {arch}") @@ -77,9 +77,7 @@ def install_deps_via_conan( ) if toolchain_file != "": load_toolchain(toolchain_file) - host_arch = get_valid_arch( - os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() - ) + host_arch = os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() host_config = [ "-s:h", f"arch={get_valid_conan_arch(host_arch)}", From 7667dde140f4d6cb53f271869bb365a1c49b3264 Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Tue, 5 Nov 2024 14:27:56 +0100 Subject: [PATCH 6/7] minor fixes --- build-system/cpp-cmake-conan/src/build.py | 2 +- build-system/cpp-cmake-conan/src/utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build-system/cpp-cmake-conan/src/build.py b/build-system/cpp-cmake-conan/src/build.py index e0b6dafd..9768cd00 100644 --- a/build-system/cpp-cmake-conan/src/build.py +++ b/build-system/cpp-cmake-conan/src/build.py @@ -72,7 +72,7 @@ def build( xcompile_toolchain_file = "" if toolchain_file != "": load_toolchain(toolchain_file) - xcompile_toolchain_file = f"-DCMAKE_TOOLCHAIN_FILE={os.path.join(safe_get_workspace_dir(),'OEToolchainConfig.cmake')}" + xcompile_toolchain_file = f"-DCMAKE_TOOLCHAIN_FILE={os.environ.get('CMAKE_TOOLCHAIN_FILE', '').strip()}" cmake_cxx_flags = f"-DCMAKE_CXX_FLAGS={os.environ.get('CXXFLAGS', '')}" host_arch = os.environ.get("OECORE_TARGET_ARCH", build_arch).strip() diff --git a/build-system/cpp-cmake-conan/src/utils.py b/build-system/cpp-cmake-conan/src/utils.py index 2c9edd50..3d58d8ad 100644 --- a/build-system/cpp-cmake-conan/src/utils.py +++ b/build-system/cpp-cmake-conan/src/utils.py @@ -58,6 +58,5 @@ def load_toolchain(toolchain_file: str) -> None: if proc.stdout is not None: for line in proc.stdout: (key, _, value) = line.decode().partition("=") - print(f"Setting {key} to {value}") os.environ[key] = value proc.communicate() From 22dcecd8f5d6b072759a0fb0124309afa824d578 Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Tue, 5 Nov 2024 14:34:35 +0100 Subject: [PATCH 7/7] strip newlines form envs --- build-system/cpp-cmake-conan/src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/cpp-cmake-conan/src/utils.py b/build-system/cpp-cmake-conan/src/utils.py index 3d58d8ad..bc307aea 100644 --- a/build-system/cpp-cmake-conan/src/utils.py +++ b/build-system/cpp-cmake-conan/src/utils.py @@ -58,5 +58,5 @@ def load_toolchain(toolchain_file: str) -> None: if proc.stdout is not None: for line in proc.stdout: (key, _, value) = line.decode().partition("=") - os.environ[key] = value + os.environ[key.strip()] = value.strip() proc.communicate()