diff --git a/bin/lib/base_library_builder.py b/bin/lib/base_library_builder.py new file mode 100644 index 000000000..9eb1d6c38 --- /dev/null +++ b/bin/lib/base_library_builder.py @@ -0,0 +1,781 @@ +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +import threading +import time +from abc import ABC, abstractmethod +from collections import defaultdict +from enum import Enum, unique +from logging import Logger +from pathlib import Path +from typing import Any + +import botocore.exceptions +import requests +from urllib3.exceptions import ProtocolError + +from lib.amazon import get_ssm_param +from lib.amazon_properties import get_specific_library_version_details +from lib.installation_context import InstallationContext +from lib.library_build_config import LibraryBuildConfig +from lib.library_platform import LibraryPlatform + +# Constants +_TIMEOUT = 30 +CONANSERVER_URL = "https://conan.compiler-explorer.com" +BUILD_TIMEOUT = 600 + +CONANINFOHASH_RE = re.compile(r"\s+ID:\s(\w*)", re.MULTILINE) + + +class PostFailure(RuntimeError): + pass + + +class FetchFailure(RuntimeError): + pass + + +@unique +class BuildStatus(Enum): + Ok = 0 + Failed = 1 + Skipped = 2 + TimedOut = 3 + + +class BaseLibraryBuilder(ABC): + """Base class for all library builders with common infrastructure.""" + + def __init__( + self, + logger: Logger, + language: str, + libname: str, + target_name: str, + sourcefolder: str, + install_context: InstallationContext, + buildconfig: LibraryBuildConfig, + platform: LibraryPlatform = LibraryPlatform.Linux, + ): + self.logger = logger + self.language = language + self.libname = libname + self.target_name = target_name + self.sourcefolder = sourcefolder + self.install_context = install_context + self.buildconfig = buildconfig + self.platform = platform + + # Common state + self.forcebuild = False + self.needs_uploading = 0 + self.conanserverproxy_token = None + self.current_commit_hash = "" + self.libid = self.libname # TODO: CE libid might be different from yaml libname + + # Build parameters + self.current_buildparameters_obj: dict[str, Any] = defaultdict(lambda: []) + self.current_buildparameters: list[str] = [] + + # Caches + self._conan_hash_cache: dict[str, str | None] = {} + self._annotations_cache: dict[str, dict] = {} + + # Thread-local HTTP session for future parallelization + self._thread_local_data = threading.local() + self._thread_local_data.session = requests.Session() + + @property + def http_session(self): + """Thread-local HTTP session.""" + return self._thread_local_data.session + + @abstractmethod + def completeBuildConfig(self): + """Complete build configuration - must be implemented by subclasses.""" + pass + + @abstractmethod + def makebuild(self, buildfor): + """Main build method - must be implemented by subclasses.""" + pass + + @abstractmethod + def makebuildfor(self, compiler, options, exe, compiler_type, toolchain, *args): + """Build for specific configuration - must be implemented by subclasses.""" + pass + + @abstractmethod + def writeconanfile(self, buildfolder): + """Write conan file - must be implemented by subclasses.""" + pass + + def get_conan_hash(self, buildfolder: str) -> str | None: + """Get conan hash for a build folder, with caching.""" + if buildfolder in self._conan_hash_cache: + self.logger.debug(f"Using cached conan hash for {buildfolder}") + return self._conan_hash_cache[buildfolder] + + if not self.install_context.dry_run: + self.logger.debug(["conan", "info", "."] + self.current_buildparameters) + conaninfo = subprocess.check_output( + ["conan", "info", "-r", "ceserver", "."] + self.current_buildparameters, cwd=buildfolder + ).decode("utf-8", "ignore") + self.logger.debug(conaninfo) + match = CONANINFOHASH_RE.search(conaninfo) + if match: + result = match[1] + self._conan_hash_cache[buildfolder] = result + return result + + self._conan_hash_cache[buildfolder] = None + return None + + def conanproxy_login(self): + """Login to conan proxy server.""" + url = f"{CONANSERVER_URL}/login" + + login_body = defaultdict(lambda: []) + if os.environ.get("CONAN_PASSWORD"): + login_body["password"] = os.environ.get("CONAN_PASSWORD") + else: + try: + login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd") + except botocore.exceptions.NoCredentialsError as exc: + raise RuntimeError( + "No password found for conan server, setup AWS credentials to access the CE SSM, or set CONAN_PASSWORD environment variable" + ) from exc + + request = self.resil_post(url, json_data=json.dumps(login_body)) + if not request.ok: + raise RuntimeError(f"Failed to login to conan proxy: {request}") + response = json.loads(request.content) + self.conanserverproxy_token = response["token"] + + def resil_post(self, url, json_data, headers=None): + """Resilient POST request with retries.""" + for _ in range(3): + try: + return self.http_session.post( + url, data=json_data, headers=headers, timeout=_TIMEOUT, allow_redirects=False + ) + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + requests.exceptions.RequestException, + ProtocolError, + ) as e: + self.logger.warning(f"Got {e} when posting to {url}, retrying") + time.sleep(1) + raise RuntimeError(f"Failed to post to {url}") + + def resil_get(self, url: str, stream: bool, timeout: int, headers=None) -> requests.Response | None: + """Resilient GET request with retries.""" + for _ in range(3): + try: + return self.http_session.get( + url, stream=stream, timeout=timeout, headers=headers, allow_redirects=False + ) + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + requests.exceptions.RequestException, + ProtocolError, + ) as e: + self.logger.warning(f"Got {e} for {url}, retrying") + time.sleep(1) + return None + + def get_build_annotations(self, buildfolder): + """Get build annotations from conan server, with caching.""" + if buildfolder in self._annotations_cache: + self.logger.debug(f"Using cached annotations for {buildfolder}") + return self._annotations_cache[buildfolder] + + conanhash = self.get_conan_hash(buildfolder) + if conanhash is None: + result = defaultdict(lambda: []) + self._annotations_cache[buildfolder] = result + return result + + url = f"{CONANSERVER_URL}/annotations/{self.libname}/{self.target_name}/{conanhash}" + with tempfile.TemporaryFile() as fd: + request = self.resil_get(url, stream=True, timeout=_TIMEOUT) + if not request or not request.ok: + raise FetchFailure(f"Fetch failure for {url}: {request}") + for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): + fd.write(chunk) + fd.flush() + fd.seek(0) + buffer = fd.read() + result = json.loads(buffer) + self._annotations_cache[buildfolder] = result + return result + + def get_commit_hash(self) -> str: + """Get the commit hash for this build.""" + if self.current_commit_hash: + return self.current_commit_hash + + if os.path.exists(f"{self.sourcefolder}/.git"): + lastcommitinfo = subprocess.check_output([ + "git", + "-C", + self.sourcefolder, + "log", + "-1", + "--oneline", + "--no-color", + ]).decode("utf-8", "ignore") + self.logger.debug(f"last git commit: {lastcommitinfo}") + match = re.match(r"^(\w*)\s.*", lastcommitinfo) + if match: + self.current_commit_hash = match[1] + else: + self.current_commit_hash = self.target_name + return self.current_commit_hash + else: + self.current_commit_hash = self.target_name + + return self.current_commit_hash + + def upload_builds(self): + """Upload builds to conan server.""" + if self.needs_uploading > 0: + if not self.install_context.dry_run: + self.logger.info("Uploading cached builds") + subprocess.check_call([ + "conan", + "upload", + f"{self.libname}/{self.target_name}", + "--all", + "-r=ceserver", + "-c", + ]) + self.logger.debug("Clearing cache to speed up next upload") + subprocess.check_call(["conan", "remove", "-f", f"{self.libname}/{self.target_name}"]) + self.needs_uploading = 0 + + def save_build_logging(self, builtok, buildfolder, extralogtext): + """Save build logging to conan server.""" + if builtok == BuildStatus.Failed: + url = f"{CONANSERVER_URL}/buildfailed" + elif builtok == BuildStatus.Ok: + url = f"{CONANSERVER_URL}/buildsuccess" + elif builtok == BuildStatus.TimedOut: + url = f"{CONANSERVER_URL}/buildfailed" + else: + return + + # Default implementation for gathering log files + # Subclasses can override to customize which logs to gather + logging_data = self._gather_build_logs(buildfolder) + + if builtok == BuildStatus.TimedOut: + logging_data = logging_data + "\n\n" + "BUILD TIMED OUT!!" + + buildparameters_copy = self.current_buildparameters_obj.copy() + buildparameters_copy["library"] = self.libname + buildparameters_copy["library_version"] = self.target_name + buildparameters_copy["logging"] = logging_data + "\n\n" + extralogtext + buildparameters_copy["commithash"] = self.get_commit_hash() + + headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} + + return self.resil_post(url, json_data=json.dumps(buildparameters_copy), headers=headers) + + def _gather_build_logs(self, buildfolder): + """Gather build log files. Override in subclasses to customize.""" + # Basic implementation - subclasses can override + return "" + + def executebuildscript(self, buildfolder): + """Execute build script.""" + scriptfile = os.path.join(buildfolder, self.script_filename) + args = self._get_script_args(scriptfile) + + self.logger.info(args) + if self.install_context.dry_run: + self.logger.info("Would run %s", args) + return BuildStatus.Ok + else: + try: + subprocess.check_call(args, cwd=buildfolder, timeout=BUILD_TIMEOUT) + return BuildStatus.Ok + except subprocess.CalledProcessError: + return BuildStatus.Failed + except subprocess.TimeoutExpired: + self.logger.error("Build timed out") + return BuildStatus.TimedOut + + def _get_script_args(self, scriptfile): + """Get script execution arguments. Override in subclasses.""" + return ["bash", scriptfile] + + def executeconanscript(self, buildfolder): + """Execute conan script with platform-specific logic.""" + if self.install_context.dry_run: + return BuildStatus.Ok + + try: + if self.platform == LibraryPlatform.Linux: + subprocess.check_call(["./conanexport.sh"], cwd=buildfolder) + elif self.platform == LibraryPlatform.Windows: + subprocess.check_call(["pwsh", "./conanexport.ps1"], cwd=buildfolder) + else: + # Fallback for other platforms + scriptfile = os.path.join(buildfolder, "conanexport.sh") + subprocess.check_call(["bash", scriptfile], cwd=buildfolder) + + self.logger.info("Export successful") + return BuildStatus.Ok + except subprocess.CalledProcessError: + return BuildStatus.Failed + + def setCurrentConanBuildParameters( + self, buildos, buildtype, compilerTypeOrGcc, compiler, libcxx, arch, stdver, extraflags + ): + """Set current conan build parameters.""" + self.current_buildparameters_obj = defaultdict(lambda: []) + self.current_buildparameters_obj["os"] = buildos + self.current_buildparameters_obj["buildtype"] = buildtype + if compilerTypeOrGcc: + self.current_buildparameters_obj["compiler"] = compilerTypeOrGcc + else: + self.current_buildparameters_obj["compiler"] = "gcc" + self.current_buildparameters_obj["compiler_version"] = compiler + if libcxx: + self.current_buildparameters_obj["libcxx"] = libcxx + else: + self.current_buildparameters_obj["libcxx"] = "libstdc++" + self.current_buildparameters_obj["arch"] = arch + self.current_buildparameters_obj["stdver"] = stdver + self.current_buildparameters_obj["flagcollection"] = extraflags + self.current_buildparameters_obj["library"] = self.libid + self.current_buildparameters_obj["library_version"] = self.target_name + + self.current_buildparameters = [ + "-s", + f"os={buildos}", + "-s", + f"build_type={buildtype}", + "-s", + f"compiler={compilerTypeOrGcc or 'gcc'}", + "-s", + f"compiler.version={compiler}", + "-s", + f"compiler.libcxx={libcxx or 'libstdc++'}", + "-s", + f"arch={arch}", + "-s", + f"stdver={stdver}", + "-s", + f"flagcollection={extraflags}", + ] + + def writeconanscript(self, buildfolder): + """Write conan export script with cross-platform support.""" + conanparamsstr = " ".join(self.current_buildparameters) + + if self.platform == LibraryPlatform.Windows: + scriptfile = Path(buildfolder) / "conanexport.ps1" + with scriptfile.open("w", encoding="utf-8") as f: + f.write(f"conan export-pkg . {self.libname}/{self.target_name} -f {conanparamsstr}\n") + else: + scriptfile = Path(buildfolder) / "conanexport.sh" + with scriptfile.open("w", encoding="utf-8") as f: + f.write("#!/bin/sh\n\n") + f.write(f"conan export-pkg . {self.libname}/{self.target_name} -f {conanparamsstr}\n") + + # Make script executable + scriptfile.chmod(0o755) + + def writebuildscript( + self, + buildfolder, + sourcefolder, + compiler, + options, + exe, + compilerType, + toolchain, + buildos, + buildtype, + arch, + stdver, + stdlib, + flagscombination, + ldPath, + compiler_props, + ): + """Write build script. This is a stub - subclasses should override.""" + # Not abstract - subclasses with different signatures can override this + return + + def makebuildhash( + self, compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination, iteration=None + ) -> str: + """Make build hash from configuration.""" + # If iteration is provided (from LibraryBuilder), use simple format + if iteration is not None: + flagsstr = "|".join(x for x in flagscombination) if flagscombination else "" + self.logger.info( + f"Building {self.libname} {self.target_name} for [{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}]" + ) + return compiler + "_" + str(iteration) + + # Otherwise use SHA256 hash (for Rust/Fortran builders) + flagsstr = "_".join(flagscombination) if flagscombination else "" + hasher = hashlib.sha256() + hasher.update(compiler.encode("utf-8", "ignore")) + hasher.update(options.encode("utf-8", "ignore")) + hasher.update(toolchain.encode("utf-8", "ignore")) + hasher.update(buildos.encode("utf-8", "ignore")) + hasher.update(buildtype.encode("utf-8", "ignore")) + hasher.update(arch.encode("utf-8", "ignore")) + hasher.update(stdver.encode("utf-8", "ignore")) + hasher.update(stdlib.encode("utf-8", "ignore")) + hasher.update(flagsstr.encode("utf-8", "ignore")) + return hasher.hexdigest() + + def build_cleanup(self, buildfolder): + """Clean up build folder after build.""" + if self.install_context.dry_run: + self.logger.info(f"Would remove directory {buildfolder} but in dry-run mode") + else: + shutil.rmtree(buildfolder, ignore_errors=True) + self.logger.info(f"Removing {buildfolder}") + + +class CompilerBasedLibraryBuilder(BaseLibraryBuilder): + """Base class for compiler-based library builders (C++, Fortran).""" + + # Class attributes for compiler popularity tracking + popular_compilers: dict[str, int] = {} + compiler_popularity_treshhold = 1000 + + def __init__( + self, + logger, + language, + libname, + target_name, + sourcefolder, + install_context, + buildconfig, + popular_compilers_only, + platform, + ): + super().__init__(logger, language, libname, target_name, sourcefolder, install_context, buildconfig, platform) + self.check_compiler_popularity = popular_compilers_only + + # These will be set by subclasses + self.compilerprops = {} + self.libraryprops = {} + + # Script filename depends on platform + self.script_filename = "cebuild.sh" + if self.platform == LibraryPlatform.Windows: + self.script_filename = "cebuild.ps1" + + def getToolchainPathFromOptions(self, options): + """Get toolchain path from compiler options.""" + match = re.search(r"--gcc-toolchain=(\S*)", options) + if match: + return match[1] + else: + # Fallback for --gxx-name option (used by some compilers) + match = re.search(r"--gxx-name=(\S*)", options) + if match: + toolchainpath = Path(match[1]).parent / ".." + return str(toolchainpath.resolve()) + return "" + + def getSysrootPathFromOptions(self, options): + """Get sysroot path from compiler options.""" + match = re.search(r"--sysroot=(\S*)", options) + if match: + return match[1] + return "" + + def getStdVerFromOptions(self, options): + """Get C++ standard version from compiler options.""" + match = re.search(r"(?:--std|-std)=(\S*)", options) + if match: + return match[1] + return "" + + def getStdLibFromOptions(self, options): + """Get stdlib from compiler options.""" + match = re.search(r"-stdlib=(\S*)", options) + if match: + return match[1] + return "" + + def getTargetFromOptions(self, options): + """Get target from compiler options.""" + # Try equals format first: --target=value or -target=value + match = re.search(r"(?:--target|-target)=(\S*)", options) + if match: + return match[1] + # Try space format: -target value + match = re.search(r"-target (\S*)", options) + if match: + return match[1] + return "" + + def get_compiler_type(self, compiler): + """Get compiler type from compiler properties.""" + if "compilerType" in self.compilerprops[compiler]: + return self.compilerprops[compiler]["compilerType"] + else: + raise RuntimeError(f"Something is wrong with {compiler}") + + def does_compiler_support(self, exe, compilerType, arch, options, ldPath): + """Check if compiler supports the given architecture.""" + if arch == "x86": + return self.does_compiler_support_x86(exe, compilerType, options, ldPath) + elif arch == "x86_64": + return self.does_compiler_support_amd64(exe, compilerType, options, ldPath) + else: + return True + + @abstractmethod + def does_compiler_support_x86(self, exe, compilerType, options, ldPath): + """Check if compiler supports x86 - must be implemented by subclasses.""" + pass + + @abstractmethod + def does_compiler_support_amd64(self, exe, compilerType, options, ldPath): + """Check if compiler supports amd64 - must be implemented by subclasses.""" + pass + + def completeBuildConfig(self): + """Complete build configuration using library properties.""" + if "description" in self.libraryprops[self.libid]: + self.buildconfig.description = self.libraryprops[self.libid]["description"] + if "name" in self.libraryprops[self.libid]: + self.buildconfig.description = self.libraryprops[self.libid]["name"] + if "url" in self.libraryprops[self.libid]: + self.buildconfig.url = self.libraryprops[self.libid]["url"] + + if "staticliblink" in self.libraryprops[self.libid]: + self.buildconfig.staticliblink = list( + set(self.buildconfig.staticliblink + self.libraryprops[self.libid]["staticliblink"]) + ) + + if "liblink" in self.libraryprops[self.libid]: + self.buildconfig.sharedliblink = list( + set(self.buildconfig.sharedliblink + self.libraryprops[self.libid]["liblink"]) + ) + + specificVersionDetails = get_specific_library_version_details(self.libraryprops, self.libid, self.target_name) + if specificVersionDetails: + if "staticliblink" in specificVersionDetails: + self.buildconfig.staticliblink = list( + set(self.buildconfig.staticliblink + specificVersionDetails["staticliblink"]) + ) + + if "liblink" in specificVersionDetails: + self.buildconfig.sharedliblink = list( + set(self.buildconfig.sharedliblink + specificVersionDetails["liblink"]) + ) + + if self.buildconfig.lib_type == "headeronly": + if self.buildconfig.staticliblink != []: + self.buildconfig.lib_type = "static" + if self.buildconfig.sharedliblink != []: + self.buildconfig.lib_type = "shared" + + if self.buildconfig.lib_type == "static": + if self.buildconfig.staticliblink == []: + self.buildconfig.staticliblink = [f"{self.libname}"] + elif self.buildconfig.lib_type == "shared": + if self.buildconfig.sharedliblink == []: + self.buildconfig.sharedliblink = [f"{self.libname}"] + elif self.buildconfig.lib_type == "cshared": + if self.buildconfig.sharedliblink == []: + self.buildconfig.sharedliblink = [f"{self.libname}"] + + if self.platform == LibraryPlatform.Windows: + self.buildconfig.package_install = True + + alternatelibs = [] + for lib in self.buildconfig.staticliblink: + if lib.endswith("d") and lib[:-1] not in self.buildconfig.staticliblink: + alternatelibs += [lib[:-1]] + else: + if f"{lib}d" not in self.buildconfig.staticliblink: + alternatelibs += [f"{lib}d"] + + self.buildconfig.staticliblink += alternatelibs + + alternatelibs = [] + for lib in self.buildconfig.sharedliblink: + if lib.endswith("d") and lib[:-1] not in self.buildconfig.sharedliblink: + alternatelibs += [lib[:-1]] + else: + if f"{lib}d" not in self.buildconfig.sharedliblink: + alternatelibs += [f"{lib}d"] + + self.buildconfig.sharedliblink += alternatelibs + + def download_compiler_usage_csv(self): + """Download compiler usage statistics from S3.""" + url = "https://compiler-explorer.s3.amazonaws.com/public/compiler_usage.csv" + with tempfile.TemporaryFile() as fd: + request = self.resil_get(url, stream=True, timeout=_TIMEOUT) + if not request or not request.ok: + raise FetchFailure(f"Fetch failure for {url}: {request}") + for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): + fd.write(chunk) + fd.flush() + fd.seek(0) + + reader = csv.DictReader(line.decode("utf-8") for line in fd.readlines()) + for row in reader: + CompilerBasedLibraryBuilder.popular_compilers[row["compiler"]] = int(row["times_used"]) + + def is_popular_enough(self, compiler): + """Check if compiler is popular enough based on usage statistics.""" + if len(CompilerBasedLibraryBuilder.popular_compilers) == 0: + self.logger.debug("downloading compiler popularity csv") + self.download_compiler_usage_csv() + + if compiler not in CompilerBasedLibraryBuilder.popular_compilers: + return False + + if ( + CompilerBasedLibraryBuilder.popular_compilers[compiler] + < CompilerBasedLibraryBuilder.compiler_popularity_treshhold + ): + return False + + return True + + def should_build_with_compiler(self, compiler, checkcompiler, buildfor): + """Check if we should build with this compiler based on filters.""" + if checkcompiler and compiler != checkcompiler: + return False + + if compiler in self.buildconfig.skip_compilers: + return False + + compilerType = self.get_compiler_type(compiler) + + exe = self.compilerprops[compiler]["exe"] + + if buildfor == "allclang" and compilerType != "clang": + return False + elif buildfor == "allicc" and "/icc" not in exe: + return False + elif buildfor == "allgcc" and compilerType: + return False + + if self.check_compiler_popularity: + if not self.is_popular_enough(compiler): + self.logger.info(f"compiler {compiler} is not popular enough") + return False + + return True + + def replace_optional_arg(self, arg, name, value): + """Replace optional argument placeholders in strings.""" + self.logger.debug(f"replace_optional_arg('{arg}', '{name}', '{value}')") + optional = "%" + name + "?%" + if optional in arg: + return arg.replace(optional, value) if value else "" + else: + return arg.replace("%" + name + "%", value) + + def expand_make_arg(self, arg, compilerTypeOrGcc, buildtype, arch, stdver, stdlib): + """Expand make argument placeholders with actual values.""" + expanded = arg + + expanded = self.replace_optional_arg(expanded, "compilerTypeOrGcc", compilerTypeOrGcc) + expanded = self.replace_optional_arg(expanded, "buildtype", buildtype) + expanded = self.replace_optional_arg(expanded, "arch", arch) + expanded = self.replace_optional_arg(expanded, "stdver", stdver) + expanded = self.replace_optional_arg(expanded, "stdlib", stdlib) + + intelarch = "" + if arch == "x86": + intelarch = "ia32" + elif arch == "x86_64": + intelarch = "intel64" + + expanded = self.replace_optional_arg(expanded, "intelarch", intelarch) + + cmake_bool_windows = "OFF" + cmake_bool_not_windows = "ON" + if self.platform == LibraryPlatform.Windows: + cmake_bool_windows = "ON" + cmake_bool_not_windows = "OFF" + + expanded = self.replace_optional_arg(expanded, "cmake_bool_not_windows", cmake_bool_not_windows) + expanded = self.replace_optional_arg(expanded, "cmake_bool_windows", cmake_bool_windows) + + return expanded + + def has_failed_before(self, build_method): + """Check if this build configuration has failed before.""" + url = f"{CONANSERVER_URL}/whathasfailedbefore" + request = self.resil_post(url, json_data=json.dumps(self.current_buildparameters_obj)) + if not request.ok: + raise PostFailure(f"Post failure for {url}: {request}") + else: + response = json.loads(request.content) + current_commit = self.get_commit_hash() + if response["commithash"] == current_commit: + return response["response"] + else: + return False + + def is_already_uploaded(self, buildfolder): + """Check if build is already uploaded to conan server.""" + conanhash = self.get_conan_hash(buildfolder) + if conanhash is None: + return False + + url = f"{CONANSERVER_URL}/conanlibrary/{self.libname}/{self.target_name}/{conanhash}" + with tempfile.TemporaryFile(): + request = self.resil_get(url, stream=True, timeout=_TIMEOUT) + if not request: + return False + return request.ok + + def set_as_uploaded(self, buildfolder, build_method): + """Mark build as uploaded to conan server.""" + conanhash = self.get_conan_hash(buildfolder) + if conanhash is None: + raise RuntimeError(f"Error determining conan hash in {buildfolder}") + + self.logger.info(f"commithash: {conanhash}") + + annotations = self.get_build_annotations(buildfolder) + if "commithash" not in annotations: + self.upload_builds() + annotations["commithash"] = self.get_commit_hash() + + # Platform-specific annotations handling would be in subclasses + self._add_platform_annotations(annotations, buildfolder) + + headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} + + url = f"{CONANSERVER_URL}/annotations/{self.libname}/{self.target_name}/{conanhash}" + request = self.resil_post(url, json_data=json.dumps(annotations), headers=headers) + if not request.ok: + raise PostFailure(f"Post failure for {url}: {request}") + + def _add_platform_annotations(self, annotations, buildfolder): + """Add platform-specific annotations. Override in subclasses if needed.""" + # Not abstract - subclasses can optionally override this + return diff --git a/bin/lib/fortran_library_builder.py b/bin/lib/fortran_library_builder.py index 7a2a13bb8..6ea04c62a 100644 --- a/bin/lib/fortran_library_builder.py +++ b/bin/lib/fortran_library_builder.py @@ -1,36 +1,28 @@ from __future__ import annotations import contextlib -import csv import glob import hashlib import itertools -import json import os import re import shutil import subprocess -import tempfile -import time from collections import defaultdict from collections.abc import Generator -from enum import Enum, unique from pathlib import Path from typing import Any, TextIO -import requests -from urllib3.exceptions import ProtocolError - -from lib.amazon import get_ssm_param -from lib.amazon_properties import get_properties_compilers_and_libraries, get_specific_library_version_details -from lib.installation_context import FetchFailure, PostFailure +from lib.amazon_properties import get_properties_compilers_and_libraries +from lib.base_library_builder import ( + BuildStatus, + CompilerBasedLibraryBuilder, +) from lib.library_build_config import LibraryBuildConfig from lib.library_platform import LibraryPlatform from lib.staging import StagingDir _TIMEOUT = 600 -compiler_popularity_treshhold = 1000 -popular_compilers: dict[str, Any] = defaultdict(lambda: []) build_supported_os = ["Linux"] build_supported_buildtype = ["Debug"] @@ -53,17 +45,7 @@ def _quote(string: str) -> str: return f'"{string}"' -@unique -class BuildStatus(Enum): - Ok = 0 - Failed = 1 - Skipped = 2 - TimedOut = 3 - - -build_timeout = 600 - -conanserver_url = "https://conan.compiler-explorer.com" +BUILD_TIMEOUT = 600 # Keep for Fortran-specific timeout if needed @contextlib.contextmanager @@ -73,7 +55,7 @@ def open_script(script: Path) -> Generator[TextIO, None, None]: script.chmod(0o755) -class FortranLibraryBuilder: +class FortranLibraryBuilder(CompilerBasedLibraryBuilder): def __init__( self, logger, @@ -85,22 +67,17 @@ def __init__( buildconfig: LibraryBuildConfig, popular_compilers_only: bool, ): - self.logger = logger - self.language = language - self.libname = libname - self.buildconfig = buildconfig - self.install_context = install_context - self.sourcefolder = sourcefolder - self.target_name = target_name - self.forcebuild = False - self.current_buildparameters_obj: dict[str, Any] = defaultdict(lambda: []) - self.current_buildparameters: list[str] = [] - self.needs_uploading = 0 - self.libid = self.libname # TODO: CE libid might be different from yaml libname - self.conanserverproxy_token = None - self._conan_hash_cache: dict[str, str | None] = {} - self._annotations_cache: dict[str, dict] = {} - self.http_session = requests.Session() + super().__init__( + logger, + language, + libname, + target_name, + sourcefolder, + install_context, + buildconfig, + popular_compilers_only, + LibraryPlatform.Linux, + ) if self.language in _propsandlibs: [self.compilerprops, self.libraryprops] = _propsandlibs[self.language] @@ -110,91 +87,30 @@ def __init__( ) _propsandlibs[self.language] = [self.compilerprops, self.libraryprops] - self.check_compiler_popularity = popular_compilers_only - self.completeBuildConfig() - def completeBuildConfig(self): - if "description" in self.libraryprops[self.libid]: - self.buildconfig.description = self.libraryprops[self.libid]["description"] - if "name" in self.libraryprops[self.libid]: - self.buildconfig.description = self.libraryprops[self.libid]["name"] - if "url" in self.libraryprops[self.libid]: - self.buildconfig.url = self.libraryprops[self.libid]["url"] - - if "staticliblink" in self.libraryprops[self.libid]: - self.buildconfig.staticliblink = list( - set(self.buildconfig.staticliblink + self.libraryprops[self.libid]["staticliblink"]) - ) - - if "liblink" in self.libraryprops[self.libid]: - self.buildconfig.sharedliblink = list( - set(self.buildconfig.sharedliblink + self.libraryprops[self.libid]["liblink"]) + def makebuildhash(self, compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination): + """Create build hash for Fortran builds with compiler prefix.""" + hasher = hashlib.sha256() + flagsstr = "|".join(x for x in flagscombination) if flagscombination else "" + hasher.update( + bytes( + f"{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}", "utf-8" ) + ) - specificVersionDetails = get_specific_library_version_details(self.libraryprops, self.libid, self.target_name) - if specificVersionDetails: - if "staticliblink" in specificVersionDetails: - self.buildconfig.staticliblink = list( - set(self.buildconfig.staticliblink + specificVersionDetails["staticliblink"]) - ) - - if "liblink" in specificVersionDetails: - self.buildconfig.sharedliblink = list( - set(self.buildconfig.sharedliblink + specificVersionDetails["liblink"]) - ) - else: - self.logger.debug("No specific library version information found") - - if self.buildconfig.lib_type == "static": - if self.buildconfig.staticliblink == []: - self.buildconfig.staticliblink = [f"{self.libname}"] - elif self.buildconfig.lib_type == "shared": - if self.buildconfig.sharedliblink == []: - self.buildconfig.sharedliblink = [f"{self.libname}"] - elif self.buildconfig.lib_type == "cshared": - if self.buildconfig.sharedliblink == []: - self.buildconfig.sharedliblink = [f"{self.libname}"] - - alternatelibs = [] - for lib in self.buildconfig.staticliblink: - if lib.endswith("d") and lib[:-1] not in self.buildconfig.staticliblink: - alternatelibs += [lib[:-1]] - else: - if f"{lib}d" not in self.buildconfig.staticliblink: - alternatelibs += [f"{lib}d"] - - self.buildconfig.staticliblink += alternatelibs - - def getToolchainPathFromOptions(self, options): - match = re.search(r"--gcc-toolchain=(\S*)", options) - if match: - return match[1] - else: - match = re.search(r"--gxx-name=(\S*)", options) - if match: - return os.path.realpath(os.path.join(os.path.dirname(match[1]), "..")) - return False - - def getStdVerFromOptions(self, options): - match = re.search(r"-std=(\S*)", options) - if match: - return match[1] - return False + self.logger.info( + f"Building {self.libname} {self.target_name} for [{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}]" + ) - def getStdLibFromOptions(self, options): - match = re.search(r"-stdlib=(\S*)", options) - if match: - return match[1] - return False + return compiler + "_" + hasher.hexdigest() - def getTargetFromOptions(self, options): - match = re.search(r"-target (\S*)", options) - if match: - return match[1] - return False + def does_compiler_support_amd64(self, exe, compilerType, options, ldPath): + """Fortran compilers generally support amd64.""" + return True - def does_compiler_support(self, exe, compilerType, arch, options, ldPath): + def _check_compiler_support_impl(self, exe, compilerType, arch, options, ldPath): + """Internal implementation of compiler support checking for FortranLibraryBuilder.""" fixedTarget = self.getTargetFromOptions(options) if fixedTarget: return fixedTarget == arch @@ -220,61 +136,67 @@ def does_compiler_support(self, exe, compilerType, arch, options, ldPath): def does_compiler_support_x86(self, exe, compilerType, options, ldPath): cachekey = f"{exe}|{options}" if cachekey not in _supports_x86: - _supports_x86[cachekey] = self.does_compiler_support(exe, compilerType, "x86", options, ldPath) + _supports_x86[cachekey] = self._check_compiler_support_impl(exe, compilerType, "x86", options, ldPath) return _supports_x86[cachekey] - def replace_optional_arg(self, arg, name, value): - optional = "%" + name + "?%" - if optional in arg: - if value: - return arg.replace(optional, value) - else: - return "" + def getToolchainPathFromOptions(self, options): + match = re.search(r"--gcc-toolchain=(\S*)", options) + if match: + return match[1] else: - return arg.replace("%" + name + "%", value) + match = re.search(r"--gxx-name=(\S*)", options) + if match: + return os.path.realpath(os.path.join(os.path.dirname(match[1]), "..")) + return False - def expand_make_arg(self, arg, compilerTypeOrGcc, buildtype, arch, stdver, stdlib): - expanded = arg + def getStdVerFromOptions(self, options): + match = re.search(r"-std=(\S*)", options) + if match: + return match[1] + return False - expanded = self.replace_optional_arg(expanded, "compilerTypeOrGcc", compilerTypeOrGcc) - expanded = self.replace_optional_arg(expanded, "buildtype", buildtype) - expanded = self.replace_optional_arg(expanded, "arch", arch) - expanded = self.replace_optional_arg(expanded, "stdver", stdver) - expanded = self.replace_optional_arg(expanded, "stdlib", stdlib) + def getStdLibFromOptions(self, options): + match = re.search(r"-stdlib=(\S*)", options) + if match: + return match[1] + return False - intelarch = "" - if arch == "x86": - intelarch = "ia32" - elif arch == "x86_64": - intelarch = "intel64" + def getTargetFromOptions(self, options): + # Align with base class pattern while supporting Fortran compilers + match = re.search(r"(?:--target|-target)[=\s](\S*)", options) + if match: + return match[1] + return False - expanded = self.replace_optional_arg(expanded, "intelarch", intelarch) + def get_compiler_type(self, compiler): + compilerType = "" + if "compilerType" in self.compilerprops[compiler]: + compilerType = self.compilerprops[compiler]["compilerType"] + else: + raise RuntimeError(f"Something is wrong with {compiler}") - return expanded + if self.compilerprops[compiler]["compilerType"] == "clang-intel": + # hack for icpx so we don't get duplicate builds + # Note: Fortran specifically needs this mapping to gcc + compilerType = "gcc" - def resil_post(self, url, json_data, headers=None): - request = None - retries = 3 - last_error = "" - while retries > 0: - try: - if headers is not None: - request = self.http_session.post(url, data=json_data, headers=headers, timeout=_TIMEOUT) - else: - request = self.http_session.post( - url, data=json_data, headers={"Content-Type": "application/json"}, timeout=_TIMEOUT - ) + return compilerType - retries = 0 - except ProtocolError as e: - last_error = e - retries = retries - 1 - time.sleep(1) + def _gather_build_logs(self, buildfolder): + """Gather Fortran-specific build logs including FPM logs.""" + logging_data = "" + # Get standard build logs + if hasattr(super(), "_gather_build_logs"): + logging_data = super()._gather_build_logs(buildfolder) - if request is None: - request = {"ok": False, "text": last_error} + # Add FPM-specific logs + fpm_logs = glob.glob(os.path.join(buildfolder, "cefpm*.txt")) + for logfile in fpm_logs: + with open(logfile, encoding="utf-8") as f: + logging_data += f"\n\n=== FPM Log: {os.path.basename(logfile)} ===\n" + logging_data += f.read() - return request + return logging_data def writebuildscript( self, @@ -354,45 +276,6 @@ def writebuildscript( buildos, buildtype, compilerTypeOrGcc, compiler, libcxx, arch, stdver, extraflags ) - def setCurrentConanBuildParameters( - self, buildos, buildtype, compilerTypeOrGcc, compiler, libcxx, arch, stdver, extraflags - ): - self.current_buildparameters_obj["os"] = buildos - self.current_buildparameters_obj["buildtype"] = buildtype - self.current_buildparameters_obj["compiler"] = compilerTypeOrGcc - self.current_buildparameters_obj["compiler_version"] = compiler - self.current_buildparameters_obj["libcxx"] = libcxx - self.current_buildparameters_obj["arch"] = arch - self.current_buildparameters_obj["stdver"] = stdver - self.current_buildparameters_obj["flagcollection"] = extraflags - self.current_buildparameters_obj["library"] = self.libid - self.current_buildparameters_obj["library_version"] = self.target_name - - self.current_buildparameters = [ - "-s", - f"os={buildos}", - "-s", - f"build_type={buildtype}", - "-s", - f"compiler={compilerTypeOrGcc}", - "-s", - f"compiler.version={compiler}", - "-s", - f"compiler.libcxx={libcxx}", - "-s", - f"arch={arch}", - "-s", - f"stdver={stdver}", - "-s", - f"flagcollection={extraflags}", - ] - - def writeconanscript(self, buildfolder): - conanparamsstr = " ".join(self.current_buildparameters) - with open_script(Path(buildfolder) / "conanexport.sh") as f: - f.write("#!/bin/sh\n\n") - f.write(f"conan export-pkg . {self.libname}/{self.target_name} -f {conanparamsstr}\n") - def write_conan_file_to(self, f: TextIO) -> None: f.write("from conans import ConanFile, tools\n") f.write(f"class {self.libname}Conan(ConanFile):\n") @@ -417,190 +300,6 @@ def writeconanfile(self, buildfolder): with (Path(buildfolder) / "conanfile.py").open(mode="w", encoding="utf-8") as f: self.write_conan_file_to(f) - def executeconanscript(self, buildfolder): - if subprocess.call(["./conanexport.sh"], cwd=buildfolder) == 0: - self.logger.info("Export succesful") - return BuildStatus.Ok - else: - return BuildStatus.Failed - - def executebuildscript(self, buildfolder): - try: - if subprocess.call(["./cebuild.sh"], cwd=buildfolder, timeout=build_timeout) == 0: - self.logger.info(f"Build succeeded in {buildfolder}") - return BuildStatus.Ok - else: - return BuildStatus.Failed - except subprocess.TimeoutExpired: - self.logger.info(f"Build timed out and was killed ({buildfolder})") - return BuildStatus.TimedOut - - def makebuildhash(self, compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination): - hasher = hashlib.sha256() - flagsstr = "|".join(x for x in flagscombination) - hasher.update( - bytes( - f"{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}", "utf-8" - ) - ) - - self.logger.info( - f"Building {self.libname} {self.target_name} for [{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}]" - ) - - return compiler + "_" + hasher.hexdigest() - - def get_conan_hash(self, buildfolder: str) -> str | None: - if buildfolder in self._conan_hash_cache: - self.logger.debug(f"Using cached conan hash for {buildfolder}") - return self._conan_hash_cache[buildfolder] - - if not self.install_context.dry_run: - self.logger.debug(["conan", "info", "."] + self.current_buildparameters) - conaninfo = subprocess.check_output( - ["conan", "info", "-r", "ceserver", "."] + self.current_buildparameters, cwd=buildfolder - ).decode("utf-8", "ignore") - self.logger.debug(conaninfo) - match = CONANINFOHASH_RE.search(conaninfo, re.MULTILINE) - if match: - result = match[1] - self._conan_hash_cache[buildfolder] = result - return result - - self._conan_hash_cache[buildfolder] = None - return None - - def conanproxy_login(self): - url = f"{conanserver_url}/login" - - login_body = defaultdict(lambda: []) - login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd") - - request = self.resil_post(url, json_data=json.dumps(login_body)) - if not request.ok: - self.logger.info(request.text) - raise PostFailure(f"Post failure for {url}: {request}") - else: - response = json.loads(request.content) - self.conanserverproxy_token = response["token"] - - def save_build_logging(self, builtok, buildfolder, extralogtext): - if builtok == BuildStatus.Failed: - url = f"{conanserver_url}/buildfailed" - elif builtok == BuildStatus.Ok: - url = f"{conanserver_url}/buildsuccess" - elif builtok == BuildStatus.TimedOut: - url = f"{conanserver_url}/buildfailed" - else: - return - - loggingfiles = [] - loggingfiles += glob.glob(buildfolder + "/cefpm*.txt") - - logging_data = "" - for logfile in loggingfiles: - logging_data += Path(logfile).read_text(encoding="utf-8") - - if builtok == BuildStatus.TimedOut: - logging_data = logging_data + "\n\n" + "BUILD TIMED OUT!!" - - buildparameters_copy = self.current_buildparameters_obj.copy() - buildparameters_copy["logging"] = logging_data + "\n\n" + extralogtext - buildparameters_copy["commithash"] = self.get_commit_hash() - - headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} - - return self.resil_post(url, json_data=json.dumps(buildparameters_copy), headers=headers) - - def get_build_annotations(self, buildfolder): - if buildfolder in self._annotations_cache: - self.logger.debug(f"Using cached annotations for {buildfolder}") - return self._annotations_cache[buildfolder] - - conanhash = self.get_conan_hash(buildfolder) - if conanhash is None: - result = defaultdict(lambda: []) - self._annotations_cache[buildfolder] = result - return result - - url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}" - with tempfile.TemporaryFile() as fd: - request = self.http_session.get(url, stream=True, timeout=_TIMEOUT) - if not request.ok: - raise FetchFailure(f"Fetch failure for {url}: {request}") - for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): - fd.write(chunk) - fd.flush() - fd.seek(0) - buffer = fd.read() - result = json.loads(buffer) - self._annotations_cache[buildfolder] = result - return result - - def get_commit_hash(self) -> str: - if os.path.exists(f"{self.sourcefolder}/.git"): - lastcommitinfo = subprocess.check_output([ - "git", - "-C", - self.sourcefolder, - "log", - "-1", - "--oneline", - "--no-color", - ]).decode("utf-8", "ignore") - self.logger.debug(lastcommitinfo) - match = GITCOMMITHASH_RE.match(lastcommitinfo) - if match: - return match[1] - else: - return self.target_name - else: - return self.target_name - - def has_failed_before(self): - url = f"{conanserver_url}/whathasfailedbefore" - request = self.resil_post(url, json_data=json.dumps(self.current_buildparameters_obj)) - if not request.ok: - raise PostFailure(f"Post failure for {url}: {request}") - else: - response = json.loads(request.content) - current_commit = self.get_commit_hash() - if response["commithash"] == current_commit: - return response["response"] - else: - return False - - def is_already_uploaded(self, buildfolder): - annotations = self.get_build_annotations(buildfolder) - - if "commithash" in annotations: - commithash = self.get_commit_hash() - - return commithash == annotations["commithash"] - else: - return False - - def set_as_uploaded(self, buildfolder): - conanhash = self.get_conan_hash(buildfolder) - if conanhash is None: - raise RuntimeError(f"Error determining conan hash in {buildfolder}") - - self.logger.info(f"commithash: {conanhash}") - - annotations = self.get_build_annotations(buildfolder) - if "commithash" not in annotations: - self.upload_builds() - annotations["commithash"] = self.get_commit_hash() - - self.logger.info(annotations) - - headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} - - url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}" - request = self.resil_post(url, json_data=json.dumps(annotations), headers=headers) - if not request.ok: - raise PostFailure(f"Post failure for {url}: {request}") - def makebuildfor( self, compiler, @@ -650,7 +349,7 @@ def makebuildfor( self.writeconanfile(build_folder) extralogtext = "" - if not self.forcebuild and self.has_failed_before(): + if not self.forcebuild and self.has_failed_before({}): self.logger.info("Build has failed before, not re-attempting") return BuildStatus.Skipped @@ -672,7 +371,7 @@ def makebuildfor( build_status = self.executeconanscript(build_folder) if build_status == BuildStatus.Ok: self.needs_uploading += 1 - self.set_as_uploaded(build_folder) + self.set_as_uploaded(build_folder, {}) if not self.install_context.dry_run: self.save_build_logging(build_status, build_folder, extralogtext) @@ -683,95 +382,6 @@ def makebuildfor( return build_status - def build_cleanup(self, buildfolder): - if self.install_context.dry_run: - self.logger.info(f"Would remove directory {buildfolder} but in dry-run mode") - else: - shutil.rmtree(buildfolder, ignore_errors=True) - self.logger.info(f"Removing {buildfolder}") - - def upload_builds(self): - if self.needs_uploading > 0: - if not self.install_context.dry_run: - self.logger.info("Uploading cached builds") - subprocess.check_call([ - "conan", - "upload", - f"{self.libname}/{self.target_name}", - "--all", - "-r=ceserver", - "-c", - ]) - self.logger.debug("Clearing cache to speed up next upload") - subprocess.check_call(["conan", "remove", "-f", f"{self.libname}/{self.target_name}"]) - self.needs_uploading = 0 - - def get_compiler_type(self, compiler): - compilerType = "" - if "compilerType" in self.compilerprops[compiler]: - compilerType = self.compilerprops[compiler]["compilerType"] - else: - raise RuntimeError(f"Something is wrong with {compiler}") - - if self.compilerprops[compiler]["compilerType"] == "clang-intel": - # hack for icpx so we don't get duplicate builds - compilerType = "gcc" - - return compilerType - - def download_compiler_usage_csv(self): - url = "https://compiler-explorer.s3.amazonaws.com/public/compiler_usage.csv" - with tempfile.TemporaryFile() as fd: - request = requests.get(url, stream=True, timeout=_TIMEOUT) - if not request.ok: - raise FetchFailure(f"Fetch failure for {url}: {request}") - for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): - fd.write(chunk) - fd.flush() - fd.seek(0) - - reader = csv.DictReader(line.decode("utf-8") for line in fd.readlines()) - for row in reader: - popular_compilers[row["compiler"]] = int(row["times_used"]) - - def is_popular_enough(self, compiler): - if len(popular_compilers) == 0: - self.logger.debug("downloading compiler popularity csv") - self.download_compiler_usage_csv() - - if compiler not in popular_compilers: - return False - - if popular_compilers[compiler] < compiler_popularity_treshhold: - return False - - return True - - def should_build_with_compiler(self, compiler, checkcompiler, buildfor): - if checkcompiler and compiler != checkcompiler: - return False - - if compiler in self.buildconfig.skip_compilers: - return False - - compilerType = self.get_compiler_type(compiler) - - exe = self.compilerprops[compiler]["exe"] - - if buildfor == "allclang" and compilerType != "clang": - return False - elif buildfor == "allicc" and "/icc" not in exe: - return False - elif buildfor == "allgcc" and compilerType: - return False - - if self.check_compiler_popularity: - if not self.is_popular_enough(compiler): - self.logger.info(f"compiler {compiler} is not popular enough") - return False - - return True - def makebuild(self, buildfor): builds_failed = 0 builds_succeeded = 0 diff --git a/bin/lib/library_builder.py b/bin/lib/library_builder.py index 5d86f7704..2791873b8 100644 --- a/bin/lib/library_builder.py +++ b/bin/lib/library_builder.py @@ -1,38 +1,29 @@ from __future__ import annotations import contextlib -import csv import glob import itertools -import json import os import re import shutil import subprocess -import tempfile -import time from collections import defaultdict from collections.abc import Generator -from enum import Enum, unique from pathlib import Path from typing import Any, TextIO -import botocore -import requests -from urllib3.exceptions import ProtocolError - -from lib.amazon import get_ssm_param -from lib.amazon_properties import get_properties_compilers_and_libraries, get_specific_library_version_details +from lib.amazon_properties import get_properties_compilers_and_libraries +from lib.base_library_builder import ( + BuildStatus, + CompilerBasedLibraryBuilder, +) from lib.binary_info import BinaryInfo -from lib.installation_context import FetchFailure, PostFailure from lib.library_build_config import LibraryBuildConfig from lib.library_build_history import LibraryBuildHistory from lib.library_platform import LibraryPlatform from lib.staging import StagingDir _TIMEOUT = 600 -compiler_popularity_treshhold = 1000 -popular_compilers: dict[str, Any] = defaultdict(lambda: []) disable_clang_libcpp = [ "clang30", @@ -88,19 +79,6 @@ def _quote(string: str) -> str: return f'"{string}"' -@unique -class BuildStatus(Enum): - Ok = 0 - Failed = 1 - Skipped = 2 - TimedOut = 3 - - -build_timeout = 600 - -conanserver_url = "https://conan.compiler-explorer.com" - - @contextlib.contextmanager def open_script(script: Path) -> Generator[TextIO, None, None]: with script.open("w", encoding="utf-8") as f: @@ -108,7 +86,7 @@ def open_script(script: Path) -> Generator[TextIO, None, None]: script.chmod(0o755) -class LibraryBuilder: +class LibraryBuilder(CompilerBasedLibraryBuilder): def __init__( self, logger, @@ -121,24 +99,25 @@ def __init__( popular_compilers_only: bool, platform: LibraryPlatform, ): - self.logger = logger - self.language = language - self.libname = libname - self.buildconfig = buildconfig - self.install_context = install_context - self.sourcefolder = sourcefolder - self.target_name = target_name - self.forcebuild = False - self.current_buildparameters_obj: dict[str, Any] = defaultdict(lambda: []) - self.current_buildparameters: list[str] = [] - self.needs_uploading = 0 - self.libid = self.libname # TODO: CE libid might be different from yaml libname - self.conanserverproxy_token = None + super().__init__( + logger, + language, + libname, + target_name, + sourcefolder, + install_context, + buildconfig, + popular_compilers_only, + platform, + ) + + # C++-specific initialization self.current_commit_hash = "" - self.platform = platform - self._conan_hash_cache: dict[str, str | None] = {} - self._annotations_cache: dict[str, dict] = {} - self.http_session = requests.Session() + + # Set script filename based on platform + self.script_filename = "cebuild.sh" + if self.platform == LibraryPlatform.Windows: + self.script_filename = "cebuild.ps1" self.history = LibraryBuildHistory(self.logger) @@ -150,117 +129,86 @@ def __init__( ) _propsandlibs[self.language] = [self.compilerprops, self.libraryprops] - self.check_compiler_popularity = popular_compilers_only - - self.script_filename = "cebuild.sh" - if self.platform == LibraryPlatform.Windows: - self.script_filename = "cebuild.ps1" - self.completeBuildConfig() - def completeBuildConfig(self): - if "description" in self.libraryprops[self.libid]: - self.buildconfig.description = self.libraryprops[self.libid]["description"] - if "name" in self.libraryprops[self.libid]: - self.buildconfig.description = self.libraryprops[self.libid]["name"] - if "url" in self.libraryprops[self.libid]: - self.buildconfig.url = self.libraryprops[self.libid]["url"] - - if "staticliblink" in self.libraryprops[self.libid]: - self.buildconfig.staticliblink = list( - set(self.buildconfig.staticliblink + self.libraryprops[self.libid]["staticliblink"]) - ) - - if "liblink" in self.libraryprops[self.libid]: - self.buildconfig.sharedliblink = list( - set(self.buildconfig.sharedliblink + self.libraryprops[self.libid]["liblink"]) - ) + def _add_platform_annotations(self, annotations, buildfolder): + """Add C++-specific platform annotations.""" + for lib in itertools.chain(self.buildconfig.staticliblink, self.buildconfig.sharedliblink): + lib_filepath = "" + if os.path.exists(os.path.join(buildfolder, f"lib{lib}.a")): + lib_filepath = os.path.join(buildfolder, f"lib{lib}.a") + elif os.path.exists(os.path.join(buildfolder, f"lib{lib}.so")): + lib_filepath = os.path.join(buildfolder, f"lib{lib}.so") + elif os.path.exists(os.path.join(buildfolder, f"{lib}.lib")): + lib_filepath = os.path.join(buildfolder, f"{lib}.lib") - specificVersionDetails = get_specific_library_version_details(self.libraryprops, self.libid, self.target_name) - if specificVersionDetails: - if "staticliblink" in specificVersionDetails: - self.buildconfig.staticliblink = list( - set(self.buildconfig.staticliblink + specificVersionDetails["staticliblink"]) - ) + if lib_filepath: + bininfo = BinaryInfo(self.logger, buildfolder, lib_filepath, self.platform) + libinfo = bininfo.cxx_info_from_binary() + archinfo = bininfo.arch_info_from_binary() + annotations["cxx11"] = libinfo["has_maybecxx11abi"] + annotations["machine"] = archinfo["elf_machine"] + if self.platform == LibraryPlatform.Windows: + annotations["osabi"] = archinfo["obj_arch"] + else: + annotations["osabi"] = archinfo["elf_osabi"] - if "liblink" in specificVersionDetails: - self.buildconfig.sharedliblink = list( - set(self.buildconfig.sharedliblink + specificVersionDetails["liblink"]) - ) + def _gather_build_logs(self, buildfolder): + """Gather C++ build log files.""" + loggingfiles = [] + loggingfiles += glob.glob(buildfolder + "/" + self.script_filename) + loggingfiles += glob.glob(buildfolder + "/cecmake*.txt") + loggingfiles += glob.glob(buildfolder + "/ceconfiglog.txt") + loggingfiles += glob.glob(buildfolder + "/cemake*.txt") + loggingfiles += glob.glob(buildfolder + "/ceinstall*.txt") + logging_data = "" + for logfile in loggingfiles: + logging_data += Path(logfile).read_text(encoding="utf-8") + return logging_data - if self.buildconfig.lib_type == "headeronly": - if self.buildconfig.staticliblink != []: - self.buildconfig.lib_type = "static" - if self.buildconfig.sharedliblink != []: - self.buildconfig.lib_type = "shared" - - if self.buildconfig.lib_type == "static": - if self.buildconfig.staticliblink == []: - self.buildconfig.staticliblink = [f"{self.libname}"] - elif self.buildconfig.lib_type == "shared": - if self.buildconfig.sharedliblink == []: - self.buildconfig.sharedliblink = [f"{self.libname}"] - elif self.buildconfig.lib_type == "cshared": - if self.buildconfig.sharedliblink == []: - self.buildconfig.sharedliblink = [f"{self.libname}"] + def save_build_logging(self, builtok, buildfolder, extralogtext): + """Override to add history tracking for C++ builds.""" + # Add history tracking + commit_hash = self.get_commit_hash() + if builtok == BuildStatus.Ok: + self.history.success(self.current_buildparameters_obj, commit_hash) + elif builtok != BuildStatus.Skipped: + self.history.failed(self.current_buildparameters_obj, commit_hash) + # Call base implementation + return super().save_build_logging(builtok, buildfolder, extralogtext) - if self.platform == LibraryPlatform.Windows: - self.buildconfig.package_install = True + def is_already_uploaded(self, buildfolder): + """Check if build is already uploaded - C++ specific implementation using annotations.""" + annotations = self.get_build_annotations(buildfolder) - alternatelibs = [] - for lib in self.buildconfig.staticliblink: - if lib.endswith("d") and lib[:-1] not in self.buildconfig.staticliblink: - alternatelibs += [lib[:-1]] - else: - if f"{lib}d" not in self.buildconfig.staticliblink: - alternatelibs += [f"{lib}d"] + if "commithash" in annotations: + commithash = self.get_commit_hash() + return commithash == annotations["commithash"] + else: + return False - self.buildconfig.staticliblink += alternatelibs + def _get_script_args(self, scriptfile): + """Get script execution arguments for C++ builds.""" + if self.platform == LibraryPlatform.Linux: + return ["./" + self.script_filename] + elif self.platform == LibraryPlatform.Windows: + return ["pwsh", "./" + self.script_filename] + else: + return ["bash", scriptfile] # fallback - alternatelibs = [] - for lib in self.buildconfig.sharedliblink: - if lib.endswith("d") and lib[:-1] not in self.buildconfig.sharedliblink: - alternatelibs += [lib[:-1]] - else: - if f"{lib}d" not in self.buildconfig.sharedliblink: - alternatelibs += [f"{lib}d"] + def get_compiler_type(self, compiler): + """Get compiler type with C++-specific clang-intel hack.""" + compilerType = "" + if "compilerType" in self.compilerprops[compiler]: + compilerType = self.compilerprops[compiler]["compilerType"] + else: + raise RuntimeError(f"Something is wrong with {compiler}") - self.buildconfig.sharedliblink += alternatelibs + if self.compilerprops[compiler]["compilerType"] == "clang-intel": + # hack for icpx so we don't get duplicate builds + compilerType = "gcc" - def getToolchainPathFromOptions(self, options): - match = re.search(r"--gcc-toolchain=(\S*)", options) - if match: - return match[1] - else: - match = re.search(r"--gxx-name=(\S*)", options) - if match: - toolchainpath = Path(match[1]).parent / ".." - return os.path.abspath(toolchainpath) - return False - - def getSysrootPathFromOptions(self, options): - match = re.search(r"--sysroot=(\S*)", options) - if match: - return match[1] - return False - - def getStdVerFromOptions(self, options): - match = re.search(r"-std=(\S*)", options) - if match: - return match[1] - return False - - def getStdLibFromOptions(self, options): - match = re.search(r"-stdlib=(\S*)", options) - if match: - return match[1] - return False - - def getTargetFromOptions(self, options): - match = re.search(r"-target (\S*)", options) - if match: - return match[1] - return False + return compilerType def getDefaultTargetFromCompiler(self, exe): try: @@ -319,7 +267,8 @@ def get_support_check_text(self, exe, compilerType, arch): return arch - def does_compiler_support(self, exe, compilerType, arch, options, ldPath): + def _check_compiler_support_impl(self, exe, compilerType, arch, options, ldPath): + """Internal implementation of compiler support checking for LibraryBuilder.""" fixedTarget = self.getTargetFromOptions(options) if fixedTarget: return fixedTarget == arch @@ -342,90 +291,11 @@ def does_compiler_support(self, exe, compilerType, arch, options, ldPath): def does_compiler_support_x86(self, exe, compilerType, options, ldPath): cachekey = f"{exe}|{options}" if cachekey not in _supports_x86: - _supports_x86[cachekey] = self.does_compiler_support(exe, compilerType, "x86", options, ldPath) + _supports_x86[cachekey] = self._check_compiler_support_impl(exe, compilerType, "x86", options, ldPath) return _supports_x86[cachekey] def does_compiler_support_amd64(self, exe, compilerType, options, ldPath): - return self.does_compiler_support(exe, compilerType, "x86_64", options, ldPath) - - def replace_optional_arg(self, arg, name, value): - self.logger.debug(f"replace_optional_arg('{arg}', '{name}', '{value}')") - optional = "%" + name + "?%" - if optional in arg: - return arg.replace(optional, value) if value else "" - else: - return arg.replace("%" + name + "%", value) - - def expand_make_arg(self, arg, compilerTypeOrGcc, buildtype, arch, stdver, stdlib): - expanded = arg - - expanded = self.replace_optional_arg(expanded, "compilerTypeOrGcc", compilerTypeOrGcc) - expanded = self.replace_optional_arg(expanded, "buildtype", buildtype) - expanded = self.replace_optional_arg(expanded, "arch", arch) - expanded = self.replace_optional_arg(expanded, "stdver", stdver) - expanded = self.replace_optional_arg(expanded, "stdlib", stdlib) - - intelarch = "" - if arch == "x86": - intelarch = "ia32" - elif arch == "x86_64": - intelarch = "intel64" - - expanded = self.replace_optional_arg(expanded, "intelarch", intelarch) - - cmake_bool_windows = "OFF" - cmake_bool_not_windows = "ON" - if self.platform == LibraryPlatform.Windows: - cmake_bool_windows = "ON" - cmake_bool_not_windows = "OFF" - - expanded = self.replace_optional_arg(expanded, "cmake_bool_not_windows", cmake_bool_not_windows) - expanded = self.replace_optional_arg(expanded, "cmake_bool_windows", cmake_bool_windows) - - return expanded - - def resil_post(self, url, json_data, headers=None): - request = None - retries = 3 - last_error = "" - while retries > 0: - try: - if headers is not None: - request = self.http_session.post(url, data=json_data, headers=headers, timeout=_TIMEOUT) - else: - request = self.http_session.post( - url, data=json_data, headers={"Content-Type": "application/json"}, timeout=_TIMEOUT - ) - - retries = 0 - except ProtocolError as e: - last_error = e - retries = retries - 1 - time.sleep(1) - - if request is None: - request = {"ok": False, "text": last_error} - - return request - - def resil_get(self, url: str, stream: bool, timeout: int, headers=None) -> requests.Response | None: - request: requests.Response | None = None - retries = 3 - while retries > 0: - try: - if headers is not None: - request = self.http_session.get(url, stream=stream, headers=headers, timeout=timeout) - else: - request = self.http_session.get( - url, stream=stream, headers={"Content-Type": "application/json"}, timeout=timeout - ) - - retries = 0 - except ProtocolError: - retries = retries - 1 - time.sleep(1) - - return request + return self._check_compiler_support_impl(exe, compilerType, "x86_64", options, ldPath) def expand_build_script_line( self, line: str, buildos, buildtype, compilerTypeOrGcc, compiler, compilerexe, libcxx, arch, stdver, extraflags @@ -812,50 +682,6 @@ def writebuildscript( buildos, buildtype, compilerTypeOrGcc, compiler, libcxx, arch, stdver, extraflags ) - def setCurrentConanBuildParameters( - self, buildos, buildtype, compilerTypeOrGcc, compiler, libcxx, arch, stdver, extraflags - ): - self.current_buildparameters_obj["os"] = buildos - self.current_buildparameters_obj["buildtype"] = buildtype - self.current_buildparameters_obj["compiler"] = compilerTypeOrGcc - self.current_buildparameters_obj["compiler_version"] = compiler - self.current_buildparameters_obj["libcxx"] = libcxx - self.current_buildparameters_obj["arch"] = arch - self.current_buildparameters_obj["stdver"] = stdver - self.current_buildparameters_obj["flagcollection"] = extraflags - self.current_buildparameters_obj["library"] = self.libid - self.current_buildparameters_obj["library_version"] = self.target_name - - self.current_buildparameters = [ - "-s", - f"os={buildos}", - "-s", - f"build_type={buildtype}", - "-s", - f"compiler={compilerTypeOrGcc}", - "-s", - f"compiler.version={compiler}", - "-s", - f"compiler.libcxx={libcxx}", - "-s", - f"arch={arch}", - "-s", - f"stdver={stdver}", - "-s", - f"flagcollection={extraflags}", - ] - - def writeconanscript(self, buildfolder): - conanparamsstr = " ".join(self.current_buildparameters) - - if self.platform == LibraryPlatform.Linux: - with open_script(Path(buildfolder) / "conanexport.sh") as f: - f.write("#!/bin/sh\n\n") - f.write(f"conan export-pkg . {self.libname}/{self.target_name} -f {conanparamsstr}\n") - elif self.platform == LibraryPlatform.Windows: - with open_script(Path(buildfolder) / "conanexport.ps1") as f: - f.write(f"conan export-pkg . {self.libname}/{self.target_name} -f {conanparamsstr}\n") - def write_conan_file_to(self, f: TextIO) -> None: libsum = ",".join( f'"{lib}"' for lib in itertools.chain(self.buildconfig.staticliblink, self.buildconfig.sharedliblink) @@ -998,245 +824,6 @@ def countValidLibraryBinaries(self, buildfolder, arch, stdlib, is_msvc: bool): return filesfound - def executeconanscript(self, buildfolder): - if self.platform == LibraryPlatform.Linux: - if subprocess.call(["./conanexport.sh"], cwd=buildfolder) == 0: - self.logger.info("Export succesful") - return BuildStatus.Ok - else: - return BuildStatus.Failed - elif self.platform == LibraryPlatform.Windows: - if subprocess.call(["pwsh", "./conanexport.ps1"], cwd=buildfolder) == 0: - self.logger.info("Export succesful") - return BuildStatus.Ok - else: - return BuildStatus.Failed - - def executebuildscript(self, buildfolder): - try: - if self.platform == LibraryPlatform.Linux: - if subprocess.call(["./" + self.script_filename], cwd=buildfolder, timeout=build_timeout) == 0: - self.logger.info(f"Build succeeded in {buildfolder}") - return BuildStatus.Ok - else: - self.logger.debug(f"Build failed in {buildfolder}") - return BuildStatus.Failed - elif self.platform == LibraryPlatform.Windows: - if subprocess.call(["pwsh", "./" + self.script_filename], cwd=buildfolder, timeout=build_timeout) == 0: - self.logger.info(f"Build succeeded in {buildfolder}") - return BuildStatus.Ok - else: - self.logger.debug(f"Build failed in {buildfolder}") - return BuildStatus.Failed - - except subprocess.TimeoutExpired: - self.logger.info(f"Build timed out and was killed ({buildfolder})") - return BuildStatus.TimedOut - - def makebuildhash( - self, compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination, iteration - ): - flagsstr = "|".join(x for x in flagscombination) - - self.logger.info( - f"Building {self.libname} {self.target_name} for [{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}]" - ) - - return compiler + "_" + str(iteration) - - def get_conan_hash(self, buildfolder: str) -> str | None: - if buildfolder in self._conan_hash_cache: - self.logger.debug(f"Using cached conan hash for {buildfolder}") - return self._conan_hash_cache[buildfolder] - - if not self.install_context.dry_run: - self.logger.debug(["conan", "info", "."] + self.current_buildparameters) - conaninfo = subprocess.check_output( - ["conan", "info", "-r", "ceserver", "."] + self.current_buildparameters, cwd=buildfolder - ).decode("utf-8", "ignore") - self.logger.debug(conaninfo) - match = CONANINFOHASH_RE.search(conaninfo, re.MULTILINE) - if match: - result = match[1] - self._conan_hash_cache[buildfolder] = result - return result - - self._conan_hash_cache[buildfolder] = None - return None - - def conanproxy_login(self): - url = f"{conanserver_url}/login" - - login_body = defaultdict(lambda: []) - if os.environ.get("CONAN_PASSWORD"): - login_body["password"] = os.environ.get("CONAN_PASSWORD") - else: - try: - login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd") - except botocore.exceptions.NoCredentialsError as exc: - raise RuntimeError( - "No password found for conan server, setup AWS credentials to access the CE SSM, or set CONAN_PASSWORD environment variable" - ) from exc - - request = self.resil_post(url, json_data=json.dumps(login_body)) - if not request.ok: - self.logger.info(request.text) - raise PostFailure(f"Post failure for {url}: {request}") - else: - response = json.loads(request.content) - self.conanserverproxy_token = response["token"] - - def save_build_logging(self, builtok, buildfolder, extralogtext): - if builtok == BuildStatus.Failed: - url = f"{conanserver_url}/buildfailed" - elif builtok == BuildStatus.Ok: - url = f"{conanserver_url}/buildsuccess" - elif builtok == BuildStatus.TimedOut: - url = f"{conanserver_url}/buildfailed" - else: - return - - loggingfiles = [] - loggingfiles += glob.glob(buildfolder + "/" + self.script_filename) - loggingfiles += glob.glob(buildfolder + "/cecmake*.txt") - loggingfiles += glob.glob(buildfolder + "/ceconfiglog.txt") - loggingfiles += glob.glob(buildfolder + "/cemake*.txt") - loggingfiles += glob.glob(buildfolder + "/ceinstall*.txt") - - logging_data = "" - for logfile in loggingfiles: - logging_data += Path(logfile).read_text(encoding="utf-8") - - if builtok == BuildStatus.TimedOut: - logging_data = logging_data + "\n\n" + "BUILD TIMED OUT!!" - - buildparameters_copy = self.current_buildparameters_obj.copy() - buildparameters_copy["logging"] = logging_data + "\n\n" + extralogtext - commit_hash = self.get_commit_hash() - buildparameters_copy["commithash"] = commit_hash - - if builtok == BuildStatus.Ok: - self.history.success(self.current_buildparameters_obj, commit_hash) - elif builtok != BuildStatus.Skipped: - self.history.failed(self.current_buildparameters_obj, commit_hash) - - headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} - - return self.resil_post(url, json_data=json.dumps(buildparameters_copy), headers=headers) - - def get_build_annotations(self, buildfolder): - if buildfolder in self._annotations_cache: - self.logger.debug(f"Using cached annotations for {buildfolder}") - return self._annotations_cache[buildfolder] - - conanhash = self.get_conan_hash(buildfolder) - if conanhash is None: - result = defaultdict(lambda: []) - self._annotations_cache[buildfolder] = result - return result - - url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}" - with tempfile.TemporaryFile() as fd: - request = self.resil_get(url, stream=True, timeout=_TIMEOUT) - if not request or not request.ok: - raise FetchFailure(f"Fetch failure for {url}: {request}") - for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): - fd.write(chunk) - fd.flush() - fd.seek(0) - buffer = fd.read() - result = json.loads(buffer) - self._annotations_cache[buildfolder] = result - return result - - def get_commit_hash(self) -> str: - if self.current_commit_hash: - return self.current_commit_hash - - if os.path.exists(f"{self.sourcefolder}/.git"): - lastcommitinfo = subprocess.check_output([ - "git", - "-C", - self.sourcefolder, - "log", - "-1", - "--oneline", - "--no-color", - ]).decode("utf-8", "ignore") - self.logger.debug(f"last git commit: {lastcommitinfo}") - match = GITCOMMITHASH_RE.match(lastcommitinfo) - if match: - self.current_commit_hash = match[1] - else: - self.current_commit_hash = self.target_name - return self.current_commit_hash - else: - self.current_commit_hash = self.target_name - - return self.current_commit_hash - - def has_failed_before(self): - url = f"{conanserver_url}/whathasfailedbefore" - request = self.resil_post(url, json_data=json.dumps(self.current_buildparameters_obj)) - if not request.ok: - raise PostFailure(f"Post failure for {url}: {request}") - else: - response = json.loads(request.content) - current_commit = self.get_commit_hash() - if response["commithash"] == current_commit: - return response["response"] - else: - return False - - def is_already_uploaded(self, buildfolder): - annotations = self.get_build_annotations(buildfolder) - - if "commithash" in annotations: - commithash = self.get_commit_hash() - - return commithash == annotations["commithash"] - else: - return False - - def set_as_uploaded(self, buildfolder): - conanhash = self.get_conan_hash(buildfolder) - if conanhash is None: - raise RuntimeError(f"Error determining conan hash in {buildfolder}") - - self.logger.info(f"commithash: {conanhash}") - - annotations = self.get_build_annotations(buildfolder) - if "commithash" not in annotations: - self.upload_builds() - annotations["commithash"] = self.get_commit_hash() - - for lib in itertools.chain(self.buildconfig.staticliblink, self.buildconfig.sharedliblink): - lib_filepath = "" - if os.path.exists(os.path.join(buildfolder, f"lib{lib}.a")): - lib_filepath = os.path.join(buildfolder, f"lib{lib}.a") - elif os.path.exists(os.path.join(buildfolder, f"lib{lib}.so")): - lib_filepath = os.path.join(buildfolder, f"lib{lib}.so") - elif os.path.exists(os.path.join(buildfolder, f"{lib}.lib")): - lib_filepath = os.path.join(buildfolder, f"{lib}.lib") - - if lib_filepath: - bininfo = BinaryInfo(self.logger, buildfolder, lib_filepath, self.platform) - libinfo = bininfo.cxx_info_from_binary() - archinfo = bininfo.arch_info_from_binary() - annotations["cxx11"] = libinfo["has_maybecxx11abi"] - annotations["machine"] = archinfo["elf_machine"] - if self.platform == LibraryPlatform.Windows: - annotations["osabi"] = archinfo["obj_arch"] - else: - annotations["osabi"] = archinfo["elf_osabi"] - - headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} - - url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}" - request = self.resil_post(url, json_data=json.dumps(annotations), headers=headers) - if not request.ok: - raise PostFailure(f"Post failure for {url}: {request}") - def makebuildfor( self, compiler, @@ -1291,7 +878,7 @@ def makebuildfor( self.writeconanfile(build_folder) extralogtext = "" - if not self.forcebuild and self.has_failed_before(): + if not self.forcebuild and self.has_failed_before({}): self.logger.info("Build has failed before, not re-attempting") return BuildStatus.Skipped @@ -1329,7 +916,7 @@ def makebuildfor( build_status = self.executeconanscript(build_folder) if build_status == BuildStatus.Ok: self.needs_uploading += 1 - self.set_as_uploaded(build_folder) + self.set_as_uploaded(build_folder, {}) else: extralogtext = "No binaries found to export" self.logger.info(extralogtext) @@ -1346,95 +933,6 @@ def makebuildfor( return build_status - def build_cleanup(self, buildfolder): - if self.install_context.dry_run: - self.logger.info(f"Would remove directory {buildfolder} but in dry-run mode") - else: - shutil.rmtree(buildfolder, ignore_errors=True) - self.logger.info(f"Removing {buildfolder}") - - def upload_builds(self): - if self.needs_uploading > 0: - if not self.install_context.dry_run: - self.logger.info("Uploading cached builds") - subprocess.check_call([ - "conan", - "upload", - f"{self.libname}/{self.target_name}", - "--all", - "-r=ceserver", - "-c", - ]) - self.logger.debug("Clearing cache to speed up next upload") - subprocess.check_call(["conan", "remove", "-f", f"{self.libname}/{self.target_name}"]) - self.needs_uploading = 0 - - def get_compiler_type(self, compiler): - compilerType = "" - if "compilerType" in self.compilerprops[compiler]: - compilerType = self.compilerprops[compiler]["compilerType"] - else: - raise RuntimeError(f"Something is wrong with {compiler}") - - if self.compilerprops[compiler]["compilerType"] == "clang-intel": - # hack for icpx so we don't get duplicate builds - compilerType = "gcc" - - return compilerType - - def download_compiler_usage_csv(self): - url = "https://compiler-explorer.s3.amazonaws.com/public/compiler_usage.csv" - with tempfile.TemporaryFile() as fd: - request = self.resil_get(url, stream=True, timeout=_TIMEOUT) - if not request or not request.ok: - raise FetchFailure(f"Fetch failure for {url}: {request}") - for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): - fd.write(chunk) - fd.flush() - fd.seek(0) - - reader = csv.DictReader(line.decode("utf-8") for line in fd.readlines()) - for row in reader: - popular_compilers[row["compiler"]] = int(row["times_used"]) - - def is_popular_enough(self, compiler): - if len(popular_compilers) == 0: - self.logger.debug("downloading compiler popularity csv") - self.download_compiler_usage_csv() - - if compiler not in popular_compilers: - return False - - if popular_compilers[compiler] < compiler_popularity_treshhold: - return False - - return True - - def should_build_with_compiler(self, compiler, checkcompiler, buildfor): - if checkcompiler and compiler != checkcompiler: - return False - - if compiler in self.buildconfig.skip_compilers: - return False - - compilerType = self.get_compiler_type(compiler) - - exe = self.compilerprops[compiler]["exe"] - - if buildfor == "allclang" and compilerType != "clang": - return False - elif buildfor == "allicc" and "/icc" not in exe: - return False - elif buildfor == "allgcc" and compilerType: - return False - - if self.check_compiler_popularity: - if not self.is_popular_enough(compiler): - self.logger.info(f"compiler {compiler} is not popular enough") - return False - - return True - def makebuild(self, buildfor): builds_failed = 0 builds_succeeded = 0 diff --git a/bin/lib/rust_library_builder.py b/bin/lib/rust_library_builder.py index f6807050e..cc4fee3aa 100644 --- a/bin/lib/rust_library_builder.py +++ b/bin/lib/rust_library_builder.py @@ -9,19 +9,14 @@ import re import shutil import subprocess -import tempfile from collections import defaultdict from collections.abc import Generator -from enum import Enum, unique from pathlib import Path from typing import Any, TextIO -import requests -from urllib3.exceptions import ProtocolError - -from lib.amazon import get_ssm_param from lib.amazon_properties import get_properties_compilers_and_libraries -from lib.installation_context import FetchFailure, InstallationContext, PostFailure +from lib.base_library_builder import CONANSERVER_URL, BaseLibraryBuilder, BuildStatus, PostFailure +from lib.installation_context import InstallationContext from lib.library_build_config import LibraryBuildConfig from lib.library_platform import LibraryPlatform from lib.rust_crates import RustCrate, get_builder_user_agent_id @@ -44,17 +39,7 @@ CONANINFOHASH_RE = re.compile(r"\s+ID:\s(\w*)") -@unique -class BuildStatus(Enum): - Ok = 0 - Failed = 1 - Skipped = 2 - TimedOut = 3 - - -build_timeout = 600 - -conanserver_url = "https://conan.compiler-explorer.com" +BUILD_TIMEOUT = 600 # Keep for Rust-specific timeout if needed @contextlib.contextmanager @@ -64,7 +49,7 @@ def open_script(script: Path) -> Generator[TextIO, None, None]: script.chmod(0o755) -class RustLibraryBuilder: +class RustLibraryBuilder(BaseLibraryBuilder): def __init__( self, logger, @@ -74,21 +59,10 @@ def __init__( install_context: InstallationContext, buildconfig: LibraryBuildConfig, ): - self.logger = logger - self.language = language - self.libname = libname - self.buildconfig = buildconfig - self.install_context = install_context - self.target_name = target_name - self.forcebuild = False - self.current_buildparameters_obj: dict[str, Any] = defaultdict(lambda: []) - self.current_buildparameters: list[str] = [] - self.needs_uploading = 0 - self.libid = self.libname # TODO: CE libid might be different from yaml libname - self.conanserverproxy_token = None - self._conan_hash_cache: dict[str, str | None] = {} - self._annotations_cache: dict[str, dict] = {} - self.http_session = requests.Session() + # Rust doesn't have sourcefolder in signature, uses Linux platform + super().__init__( + logger, language, libname, target_name, "", install_context, buildconfig, LibraryPlatform.Linux + ) if self.language in _propsandlibs: [self.compilerprops, self.libraryprops] = _propsandlibs[self.language] @@ -102,6 +76,10 @@ def __init__( self.completeBuildConfig() + @property + def script_filename(self): + return "cebuild.sh" + def completeBuildConfig(self): if "description" in self.libraryprops[self.libid]: self.buildconfig.description = self.libraryprops[self.libid]["description"] @@ -153,7 +131,7 @@ def writebuildscript( cargoline = f"$CARGO build {methodflags} --target-dir {buildfolder} > {logfolder}/buildlog.txt 2>&1\n" f.write(cargoline) else: - raise RuntimeError("Unknown build_type {self.buildconfig.build_type}") + raise RuntimeError(f"Unknown build_type {self.buildconfig.build_type}") self.setCurrentConanBuildParameters( buildos, buildtype, compilerType, compiler, libcxx, arch, stdver, extraflags @@ -162,6 +140,8 @@ def writebuildscript( def setCurrentConanBuildParameters( self, buildos, buildtype, compilerTypeOrGcc, compiler, libcxx, arch, stdver, extraflags ): + """Set current conan build parameters for Rust builds.""" + self.current_buildparameters_obj = {} self.current_buildparameters_obj["os"] = buildos self.current_buildparameters_obj["buildtype"] = buildtype self.current_buildparameters_obj["compiler"] = compilerTypeOrGcc @@ -183,20 +163,29 @@ def setCurrentConanBuildParameters( "-s", f"compiler.version={compiler}", "-s", - f"compiler.libcxx={libcxx}", - "-s", f"arch={arch}", "-s", f"stdver={stdver}", "-s", f"flagcollection={extraflags}", ] + self.current_buildparameters.extend(["-s", f"compiler.libcxx={libcxx}"]) - def writeconanscript(self, buildfolder): - conanparamsstr = " ".join(self.current_buildparameters) - with open_script(Path(buildfolder) / "conanexport.sh") as f: - f.write("#!/bin/sh\n\n") - f.write(f"conan export-pkg . {self.libname}/{self.target_name} -f {conanparamsstr}\n") + def makebuildhash(self, compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination): + """Create build hash for Rust builds with compiler prefix.""" + hasher = hashlib.sha256() + flagsstr = "|".join(x for x in flagscombination) if flagscombination else "" + hasher.update( + bytes( + f"{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}", "utf-8" + ) + ) + + self.logger.info( + f"Building {self.libname} {self.target_name} for [{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}]" + ) + + return compiler + "_" + hasher.hexdigest() def writeconanfile(self, buildfolder): underscoredlibname = self.libname.replace("-", "_") @@ -219,21 +208,19 @@ def countValidLibraryBinaries(self, buildfolder, arch, stdlib): return filesfound - def executeconanscript(self, buildfolder, arch, stdlib): + def validate_and_export_conan(self, buildfolder, arch, stdlib): + """Validate Rust binaries and export via conan if valid.""" filesfound = self.countValidLibraryBinaries(buildfolder, arch, stdlib) if filesfound != 0: - if subprocess.call(["./conanexport.sh"], cwd=buildfolder) == 0: - self.logger.info("Export succesful") - return BuildStatus.Ok - else: - return BuildStatus.Failed + # Use base class executeconanscript for actual execution + return self.executeconanscript(buildfolder) else: self.logger.info("No binaries found to export") return BuildStatus.Failed def executebuildscript(self, buildfolder): try: - if subprocess.call(["./build.sh"], cwd=buildfolder, timeout=build_timeout) == 0: + if subprocess.call(["./build.sh"], cwd=buildfolder, timeout=BUILD_TIMEOUT) == 0: self.logger.info(f"Build succeeded in {buildfolder}") return BuildStatus.Ok else: @@ -242,87 +229,13 @@ def executebuildscript(self, buildfolder): self.logger.info(f"Build timed out and was killed ({buildfolder})") return BuildStatus.TimedOut - def makebuildhash(self, compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination): - hasher = hashlib.sha256() - flagsstr = "|".join(x for x in flagscombination) - hasher.update( - bytes( - f"{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}", "utf-8" - ) - ) - - self.logger.info( - f"Building {self.libname} {self.target_name} for [{compiler},{options},{toolchain},{buildos},{buildtype},{arch},{stdver},{stdlib},{flagsstr}]" - ) - - return compiler + "_" + hasher.hexdigest() - - def get_conan_hash(self, buildfolder: str) -> str | None: - if buildfolder in self._conan_hash_cache: - self.logger.debug(f"Using cached conan hash for {buildfolder}") - return self._conan_hash_cache[buildfolder] - - if not self.install_context.dry_run: - self.logger.debug(["conan", "info", "."] + self.current_buildparameters) - conaninfo = subprocess.check_output( - ["conan", "info", "-r", "ceserver", "."] + self.current_buildparameters, cwd=buildfolder - ).decode("utf-8", "ignore") - self.logger.debug(conaninfo) - match = CONANINFOHASH_RE.search(conaninfo, re.MULTILINE) - if match: - result = match[1] - self._conan_hash_cache[buildfolder] = result - return result - - self._conan_hash_cache[buildfolder] = None - return None - - def resil_post(self, url, json_data, headers=None): - request = None - retries = 3 - last_error = "" - while retries > 0: - try: - if headers is not None: - request = self.http_session.post(url, data=json_data, headers=headers, timeout=_TIMEOUT) - else: - request = self.http_session.post( - url, data=json_data, headers={"Content-Type": "application/json"}, timeout=_TIMEOUT - ) - - retries = 0 - except ProtocolError as e: - last_error = e - retries = retries - 1 - - if request is None: - request = {"ok": False, "text": last_error} - - return request - - def conanproxy_login(self): - url = f"{conanserver_url}/login" - - login_body = defaultdict(lambda: []) - login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd") - - req_data = json.dumps(login_body) - - request = self.resil_post(url, req_data) - if not request.ok: - self.logger.info(request.text) - raise RuntimeError(f"Post failure for {url}: {request}") - else: - response = json.loads(request.content) - self.conanserverproxy_token = response["token"] - def save_build_logging(self, builtok, logfolder, source_folder, build_method): if builtok == BuildStatus.Failed: - url = f"{conanserver_url}/buildfailed" + url = f"{CONANSERVER_URL}/buildfailed" elif builtok == BuildStatus.Ok: - url = f"{conanserver_url}/buildsuccess" + url = f"{CONANSERVER_URL}/buildsuccess" elif builtok == BuildStatus.TimedOut: - url = f"{conanserver_url}/buildfailed" + url = f"{CONANSERVER_URL}/buildfailed" else: return @@ -350,61 +263,45 @@ def save_build_logging(self, builtok, logfolder, source_folder, build_method): if not request.ok: raise PostFailure(f"Post failure for {url}: {request}") - def get_build_annotations(self, buildfolder): - if buildfolder in self._annotations_cache: - self.logger.debug(f"Using cached annotations for {buildfolder}") - return self._annotations_cache[buildfolder] - - conanhash = self.get_conan_hash(buildfolder) - if conanhash is None: - result = defaultdict(lambda: []) - self._annotations_cache[buildfolder] = result - return result - - url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}" - with tempfile.TemporaryFile() as fd: - request = self.http_session.get(url, stream=True, timeout=_TIMEOUT) - if not request.ok: - raise FetchFailure(f"Fetch failure for {url}: {request}") - for chunk in request.iter_content(chunk_size=4 * 1024 * 1024): - fd.write(chunk) - fd.flush() - fd.seek(0) - buffer = fd.read() - result = json.loads(buffer) - self._annotations_cache[buildfolder] = result - return result - def get_commit_hash(self) -> str: return self.target_name def has_failed_before(self, build_method): - url = f"{conanserver_url}/hasfailedbefore" - - data = self.current_buildparameters_obj.copy() - data["flagcollection"] = build_method["build_method"] - - req_data = json.dumps(data) - - request = self.resil_post(url, req_data) - if not request.ok: - raise PostFailure(f"Post failure for {url}: {request}") + """Check if this build configuration has failed before. + + Rust override to handle build_method parameter. + """ + if build_method: + # Rust-specific: check with build method + url = f"{CONANSERVER_URL}/hasfailedbefore" + data = self.current_buildparameters_obj.copy() + data["flagcollection"] = build_method["build_method"] + req_data = json.dumps(data) + request = self.resil_post(url, req_data) + if not request.ok: + raise PostFailure(f"Post failure for {url}: {request}") + else: + response = json.loads(request.content) + return response["response"] else: - response = json.loads(request.content) - return response["response"] + # Fall back to base implementation + return super().has_failed_before({}) - def is_already_uploaded(self, buildfolder, source_folder): + def is_already_uploaded(self, buildfolder): + """Check if build is already uploaded. + + Rust override that checks annotations for commithash. + """ annotations = self.get_build_annotations(buildfolder) self.logger.debug("Annotations: " + json.dumps(annotations)) if "commithash" in annotations: commithash = self.get_commit_hash() - return commithash == annotations["commithash"] else: return False - def set_as_uploaded(self, buildfolder, source_folder, build_method): + def set_as_uploaded(self, buildfolder, build_method): conanhash = self.get_conan_hash(buildfolder) if conanhash is None: raise RuntimeError(f"Error determining conan hash in {buildfolder}") @@ -416,13 +313,14 @@ def set_as_uploaded(self, buildfolder, source_folder, build_method): self.upload_builds() annotations["commithash"] = self.get_commit_hash() - for key in build_method: - annotations[key] = build_method[key] + if build_method: + for key in build_method: + annotations[key] = build_method[key] self.logger.info(annotations) headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} - url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}" + url = f"{CONANSERVER_URL}/annotations/{self.libname}/{self.target_name}/{conanhash}" request = self.resil_post(url, json.dumps(annotations), headers) if not request.ok: @@ -575,7 +473,7 @@ def makebuildfor_by_method( self.logger.info("Build has failed before, not re-attempting") return BuildStatus.Skipped - if self.is_already_uploaded(build_folder, source_folder): + if self.is_already_uploaded(build_folder): self.logger.info("Build already uploaded") if not self.forcebuild: return BuildStatus.Skipped @@ -589,10 +487,10 @@ def makebuildfor_by_method( if build_status == BuildStatus.Ok: self.writeconanscript(build_folder) if not self.install_context.dry_run: - build_status = self.executeconanscript(build_folder, arch, stdlib) + build_status = self.validate_and_export_conan(build_folder, arch, stdlib) if build_status == BuildStatus.Ok: self.needs_uploading += 1 - self.set_as_uploaded(build_folder, source_folder, build_method) + self.set_as_uploaded(build_folder, build_method) else: filesfound = self.countValidLibraryBinaries(build_folder, arch, stdlib) self.logger.debug(f"Number of valid library binaries {filesfound}") @@ -609,13 +507,6 @@ def makebuildfor_by_method( return build_status - def build_cleanup(self, buildfolder): - if self.install_context.dry_run: - self.logger.info(f"Would remove directory {buildfolder} but in dry-run mode") - else: - shutil.rmtree(buildfolder, ignore_errors=True) - self.logger.info(f"Removing {buildfolder}") - def cache_cleanup(self): if not self.install_context.dry_run: for folder in self.cached_source_folders: @@ -623,22 +514,6 @@ def cache_cleanup(self): else: self.logger.info("Would clean crate cache, but in dry-run mode") - def upload_builds(self): - if self.needs_uploading > 0: - if not self.install_context.dry_run: - self.logger.info("Uploading cached builds") - subprocess.check_call([ - "conan", - "upload", - f"{self.libname}/{self.target_name}", - "--all", - "-r=ceserver", - "-c", - ]) - self.logger.debug("Clearing cache to speed up next upload") - subprocess.check_call(["conan", "remove", "-f", f"{self.libname}/{self.target_name}"]) - self.needs_uploading = 0 - def makebuild(self, buildfor): builds_failed = 0 builds_succeeded = 0 diff --git a/bin/test/base_library_builder_test.py b/bin/test/base_library_builder_test.py new file mode 100644 index 000000000..7072c6902 --- /dev/null +++ b/bin/test/base_library_builder_test.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from lib.base_library_builder import CONANINFOHASH_RE, BaseLibraryBuilder +from lib.library_build_config import LibraryBuildConfig + + +@pytest.fixture +def test_builder(): + """Create a test builder instance.""" + logger = MagicMock() + install_context = MagicMock() + install_context.dry_run = False + buildconfig = LibraryBuildConfig({ + "lib_type": "static", + "package_dir": "libs", + "source_archive": "", + }) + + # Create a concrete test class since BaseLibraryBuilder is abstract + class TestBuilder(BaseLibraryBuilder): + def completeBuildConfig(self): + pass + + def makebuild(self, buildfor): + pass + + def makebuildfor(self, *args): + pass + + def writeconanfile(self, buildfolder): + pass + + return TestBuilder( + logger=logger, + language="c++", + libname="testlib", + target_name="1.0", + sourcefolder="/tmp/test", + install_context=install_context, + buildconfig=buildconfig, + ) + + +def test_setCurrentConanBuildParameters_format(test_builder): + """Test that conan parameters are built in the correct format.""" + # Call the method with typical parameters + test_builder.setCurrentConanBuildParameters( + buildos="Linux", + buildtype="Debug", + compilerTypeOrGcc="gcc", + compiler="11.1.0", + libcxx="libstdc++", + arch="x86_64", + stdver="c++17", + extraflags="", + ) + + # Expected format: alternating "-s" and "key=value" as separate list items + expected = [ + "-s", + "os=Linux", + "-s", + "build_type=Debug", + "-s", + "compiler=gcc", + "-s", + "compiler.version=11.1.0", + "-s", + "compiler.libcxx=libstdc++", + "-s", + "arch=x86_64", + "-s", + "stdver=c++17", + "-s", + "flagcollection=", + ] + + assert test_builder.current_buildparameters == expected + + +def test_setCurrentConanBuildParameters_with_defaults(test_builder): + """Test parameter building with None/empty values that trigger defaults.""" + test_builder.setCurrentConanBuildParameters( + buildos="Linux", + buildtype="Release", + compilerTypeOrGcc=None, # Should default to "gcc" + compiler="10.2.0", + libcxx=None, # Should default to "libstdc++" + arch="x86", + stdver="", + extraflags="", + ) + + expected = [ + "-s", + "os=Linux", + "-s", + "build_type=Release", + "-s", + "compiler=gcc", # defaulted + "-s", + "compiler.version=10.2.0", + "-s", + "compiler.libcxx=libstdc++", # defaulted + "-s", + "arch=x86", + "-s", + "stdver=", + "-s", + "flagcollection=", + ] + + assert test_builder.current_buildparameters == expected + + +def test_setCurrentConanBuildParameters_includes_flagcollection(test_builder): + """Test that flagcollection is included in conan parameters list.""" + test_builder.setCurrentConanBuildParameters( + buildos="Linux", + buildtype="Debug", + compilerTypeOrGcc="clang", + compiler="14.0.0", + libcxx="libc++", + arch="x86_64", + stdver="c++20", + extraflags="-O3", + ) + + # Verify flagcollection is in the object + assert test_builder.current_buildparameters_obj["flagcollection"] == "-O3" + + # Verify it IS in the parameter list (original behavior) + param_strings = " ".join(test_builder.current_buildparameters) + assert "flagcollection=-O3" in param_strings + + +def test_conan_parameter_format_for_command_line(test_builder): + """Test that parameters can be used directly with conan command.""" + test_builder.setCurrentConanBuildParameters("Linux", "Debug", "gcc", "11.1.0", "libstdc++", "x86_64", "c++17", "") + + # Simulate building a conan command + conan_cmd = ["conan", "info", "."] + test_builder.current_buildparameters + + # Verify the command would be correctly formatted + expected_cmd = [ + "conan", + "info", + ".", + "-s", + "os=Linux", + "-s", + "build_type=Debug", + "-s", + "compiler=gcc", + "-s", + "compiler.version=11.1.0", + "-s", + "compiler.libcxx=libstdc++", + "-s", + "arch=x86_64", + "-s", + "stdver=c++17", + "-s", + "flagcollection=", + ] + + assert conan_cmd == expected_cmd + + +def test_conan_hash_regex_with_real_output(): + """Test that the regex pattern matches actual conan info output.""" + # Test real conan info output format (based on what's expected from conan) + conan_output = """ +[project/1.0@celibs/trunk] + ID: 5ab84d28a1f62d3983d85f5b69d0e4e45741e7e9 + BuildID: None + Context: host + """ + + match = CONANINFOHASH_RE.search(conan_output) + assert match is not None + assert match.group(1) == "5ab84d28a1f62d3983d85f5b69d0e4e45741e7e9" + + +@pytest.mark.parametrize( + "text,expected_hash", + [ + # Standard conan output formats we expect + (" ID: abc123def456", "abc123def456"), + ("\tID: 789abc", "789abc"), + (" ID: 5ab84d28a1f62d3983d85f5b69d0e4e45741e7e9", "5ab84d28a1f62d3983d85f5b69d0e4e45741e7e9"), + # Edge case: empty hash (allowed by \w*) + (" ID: ", ""), + ], +) +def test_conan_hash_regex_valid_patterns(text, expected_hash): + """Test regex matches valid ID patterns from conan output.""" + match = CONANINFOHASH_RE.search(text) + assert match is not None + assert match.group(1) == expected_hash + + +@pytest.mark.parametrize( + "text", + [ + "ID: abc123", # No whitespace before ID: + " ID:abc123", # No whitespace after ID: + ], +) +def test_conan_hash_regex_invalid_patterns(text): + """Test regex correctly rejects invalid patterns.""" + match = CONANINFOHASH_RE.search(text) + assert match is None diff --git a/bin/test/fortran_library_builder_test.py b/bin/test/fortran_library_builder_test.py index 2115a6c25..3691b6874 100644 --- a/bin/test/fortran_library_builder_test.py +++ b/bin/test/fortran_library_builder_test.py @@ -189,7 +189,8 @@ def test_get_conan_hash_success(mock_subprocess, requests_mock): def test_execute_build_script_success(mock_subprocess, requests_mock): requests_mock.get(f"{BASE}fortran.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) - install_context = mock.Mock(spec_set=InstallationContext) + install_context = mock.Mock() + install_context.dry_run = False build_config = create_fortran_test_build_config() builder = FortranLibraryBuilder( logger, "fortran", "fortranlib", "2.0.0", "/tmp/source", install_context, build_config, False @@ -200,14 +201,17 @@ def test_execute_build_script_success(mock_subprocess, requests_mock): result = builder.executebuildscript("/tmp/buildfolder") assert result == BuildStatus.Ok - mock_subprocess.assert_called_once_with(["./cebuild.sh"], cwd="/tmp/buildfolder", timeout=600) + mock_subprocess.assert_called_once_with( + ["bash", "/tmp/buildfolder/cebuild.sh"], cwd="/tmp/buildfolder", timeout=600 + ) @patch("subprocess.call") def test_execute_build_script_timeout(mock_subprocess, requests_mock): requests_mock.get(f"{BASE}fortran.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) - install_context = mock.Mock(spec_set=InstallationContext) + install_context = mock.Mock() + install_context.dry_run = False build_config = create_fortran_test_build_config() builder = FortranLibraryBuilder( logger, "fortran", "fortranlib", "2.0.0", "/tmp/source", install_context, build_config, False @@ -220,7 +224,7 @@ def test_execute_build_script_timeout(mock_subprocess, requests_mock): assert result == BuildStatus.TimedOut -@patch("lib.fortran_library_builder.get_ssm_param") +@patch("lib.base_library_builder.get_ssm_param") def test_conanproxy_login_success(mock_get_ssm, requests_mock): requests_mock.get(f"{BASE}fortran.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) diff --git a/bin/test/library_builder_test.py b/bin/test/library_builder_test.py index d2ea873c5..c706e1337 100644 --- a/bin/test/library_builder_test.py +++ b/bin/test/library_builder_test.py @@ -124,7 +124,7 @@ def test_get_toolchain_path_from_options_none(requests_mock): options = "-O2 -std=c++17" result = builder.getToolchainPathFromOptions(options) - assert result is False + assert not result def test_get_sysroot_path_from_options(requests_mock): @@ -251,7 +251,8 @@ def test_get_conan_hash_success(mock_subprocess, requests_mock): def test_execute_build_script_success(mock_subprocess, requests_mock): requests_mock.get(f"{BASE}cpp.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) - install_context = mock.Mock(spec_set=InstallationContext) + install_context = mock.Mock() + install_context.dry_run = False build_config = create_test_build_config() builder = LibraryBuilder( logger, "cpp", "testlib", "1.0.0", "/tmp/source", install_context, build_config, False, LibraryPlatform.Linux @@ -269,7 +270,8 @@ def test_execute_build_script_success(mock_subprocess, requests_mock): def test_execute_build_script_timeout(mock_subprocess, requests_mock): requests_mock.get(f"{BASE}cpp.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) - install_context = mock.Mock(spec_set=InstallationContext) + install_context = mock.Mock() + install_context.dry_run = False build_config = create_test_build_config() builder = LibraryBuilder( logger, "cpp", "testlib", "1.0.0", "/tmp/source", install_context, build_config, False, LibraryPlatform.Linux @@ -282,7 +284,7 @@ def test_execute_build_script_timeout(mock_subprocess, requests_mock): assert result == BuildStatus.TimedOut -@patch("lib.library_builder.get_ssm_param") +@patch("lib.base_library_builder.get_ssm_param") def test_conanproxy_login_success(mock_get_ssm, requests_mock): requests_mock.get(f"{BASE}cpp.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) @@ -305,7 +307,7 @@ def test_conanproxy_login_success(mock_get_ssm, requests_mock): mock_get_ssm.assert_called_once_with("/compiler-explorer/conanpwd") -@patch("lib.library_builder.get_ssm_param") +@patch("lib.base_library_builder.get_ssm_param") def test_conanproxy_login_with_env_var(mock_get_ssm, requests_mock): requests_mock.get(f"{BASE}cpp.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) diff --git a/bin/test/rust_library_builder_test.py b/bin/test/rust_library_builder_test.py index dfb33d9b5..588299a58 100644 --- a/bin/test/rust_library_builder_test.py +++ b/bin/test/rust_library_builder_test.py @@ -105,7 +105,7 @@ def test_execute_build_script_timeout(mock_subprocess, requests_mock): assert result == BuildStatus.TimedOut -@patch("lib.rust_library_builder.get_ssm_param") +@patch("lib.base_library_builder.get_ssm_param") def test_conanproxy_login_success(mock_get_ssm, requests_mock): requests_mock.get(f"{BASE}rust.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) @@ -141,13 +141,14 @@ def test_get_commit_hash(requests_mock): def test_execute_conan_script_success(mock_subprocess, requests_mock): requests_mock.get(f"{BASE}rust.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) - install_context = mock.Mock(spec_set=InstallationContext) + install_context = mock.Mock() + install_context.dry_run = False build_config = create_rust_test_build_config() builder = RustLibraryBuilder(logger, "rust", "rustlib", "1.0.0", install_context, build_config) mock_subprocess.return_value = 0 - result = builder.executeconanscript("/tmp/buildfolder", "x86_64", "") + result = builder.validate_and_export_conan("/tmp/buildfolder", "x86_64", "") assert result == BuildStatus.Ok mock_subprocess.assert_called_once_with(["./conanexport.sh"], cwd="/tmp/buildfolder") @@ -157,13 +158,14 @@ def test_execute_conan_script_success(mock_subprocess, requests_mock): def test_execute_conan_script_failure(mock_subprocess, requests_mock): requests_mock.get(f"{BASE}rust.amazon.properties", text="") logger = mock.Mock(spec_set=Logger) - install_context = mock.Mock(spec_set=InstallationContext) + install_context = mock.Mock() + install_context.dry_run = False build_config = create_rust_test_build_config() builder = RustLibraryBuilder(logger, "rust", "rustlib", "1.0.0", install_context, build_config) mock_subprocess.return_value = 1 - result = builder.executeconanscript("/tmp/buildfolder", "x86_64", "") + result = builder.validate_and_export_conan("/tmp/buildfolder", "x86_64", "") assert result == BuildStatus.Failed