From f45ff675d3831426d1fe3d53d8d85d542217914c Mon Sep 17 00:00:00 2001 From: David Candler Date: Wed, 2 Oct 2024 14:54:06 +0100 Subject: [PATCH 1/5] Add additional patch folder for performance patches This adds a second llvm-project patch directory, intended for performance enhancement patches which can be kept optional to avoid blocking builds if they become out of date with the main branch. A new CMake option controls whether or not to apply the patches, and a new script is included to manage the patching process to make it easier to identify when there is a problem with a patch. --- CMakeLists.txt | 24 ++-- cmake/patch_llvm.py | 132 ++++++++++++++++++ .../0000-Placeholder-commit.patch | 22 +++ 3 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 cmake/patch_llvm.py create mode 100644 patches/llvm-project-perf/0000-Placeholder-commit.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index eaf4fd23..f0a31de7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,11 @@ option( the tools every time you update llvm-project." ON ) +option( + APPLY_LLVM_PERFORMANCE_PATCHES + "During checkout, apply optional downstream patches to + llvm-project to improve performance." +) set( FVP_INSTALL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/fvp/install" CACHE STRING @@ -280,24 +285,17 @@ if(NOT (LLVM_TOOLCHAIN_C_LIBRARY STREQUAL llvmlibc)) # libc in a separate repo? read_repo_version(${LLVM_TOOLCHAIN_C_LIBRARY} ${LLVM_TOOLCHAIN_C_LIBRARY}) endif() -# The patches are generated from custom branch, with followin command: -# git format-patch -k origin/main -# The patches apply cleanly against llvm-project commit -# 327124ece7d59de56ca0f9faa2cd82af68c011b9. -set( - llvm_project_patches - ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project/0001-libc-tests-with-picolibc-xfail-one-remaining-test.patch - ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project/0002-libc-tests-with-picolibc-disable-large-tests.patch - ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project/0003-Disable-failing-compiler-rt-test.patch - ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project/0004-libc-tests-with-picolibc-XFAIL-uses-of-atomics.patch - ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project/0005-libc-tests-with-picolibc-mark-two-more-large-tests.patch -) +set(LLVM_PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch_llvm.py ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project) +if(APPLY_LLVM_PERFORMANCE_PATCHES) + set(LLVM_PATCH_COMMAND ${LLVM_PATCH_COMMAND} && ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch_llvm.py ${CMAKE_CURRENT_SOURCE_DIR}/patches/llvm-project-perf) +endif() + FetchContent_Declare(llvmproject GIT_REPOSITORY https://github.com/llvm/llvm-project.git GIT_TAG "${llvmproject_TAG}" GIT_SHALLOW "${llvmproject_SHALLOW}" GIT_PROGRESS TRUE - PATCH_COMMAND git reset --quiet --hard && git clean --quiet --force -dx && git am -k --ignore-whitespace --3way ${llvm_project_patches} + PATCH_COMMAND ${LLVM_PATCH_COMMAND} # Add the llvm subdirectory later to ensure that # LLVMEmbeddedToolchainForArm is the first project declared. # Otherwise CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT diff --git a/cmake/patch_llvm.py b/cmake/patch_llvm.py new file mode 100644 index 00000000..4a9e2092 --- /dev/null +++ b/cmake/patch_llvm.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +""" +Script to apply a set of patches to llvm-project sources. +""" + +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "patchdir", + help="Set of patches to apply. This should be a directory containing one or more ordered *.patch files.", + ) + parser.add_argument( + "--llvm_dir", + help="Directory of the llvm-project git checkout, if not the current directory.", + ) + parser.add_argument( + "--method", + choices=["am", "apply"], + default="apply", + help="Git command to use. git am will add each patch as a commit, whereas git apply will leave patched changes staged.", + ) + parser.add_argument( + "--reset", + help="Clean and reset the repo to a specified commit before patching.", + ) + parser.add_argument( + "--restore_on_fail", + action="store_true", + help="If a patch in a series cannot be applied, restore the original state instead of leaving patches missing. Return code will be 2 instead of 1.", + ) + args = parser.parse_args() + + if args.reset: + reset_args = ["git", "reset", "--quiet", "--hard"] + subprocess.check_output(reset_args, cwd=args.llvm_dir) + clean_args = ["git", "clean", "--quiet", "--force", "-dx"] + subprocess.check_output(clean_args, cwd=args.llvm_dir) + + patch_names = [] + for patch_name in os.listdir(args.patchdir): + if patch_name.endswith(".patch"): + patch_names.append(patch_name) + patch_names.sort() + + print(f"Found {len(patch_names)} patches to apply:") + for patch_name in patch_names: + print(patch_name) + + if args.method == "am": + merge_args = ["git", "am", "-k", "--ignore-whitespace", "--3way"] + for patch_name in patch_names: + merge_args.append(os.path.join(args.patchdir, patch_name)) + p = subprocess.run( + merge_args, cwd=args.llvm_dir, capture_output=True, text=True + ) + print(p.stdout) + print(p.stderr) + + if p.returncode == 0: + print(f"All patches applied.") + sys.exit(0) + if args.restore_on_fail: + # Check that the operation can be aborted. + if ( + 'To restore the original branch and stop patching, run "git am --abort".' + in p.stdout + ): + print("Aborting git am...") + subprocess.run(["git", "am", "--abort"], cwd=args.llvm_dir, check=True) + sys.exit(2) + sys.exit(1) + else: + applied_patches = [] + for patch_name in patch_names: + patch_file = os.path.join(args.patchdir, patch_name) + print(f"Checking {patch_file}...") + # Check that the patch applies before trying to apply it. + apply_check_args = [ + "git", + "apply", + "--ignore-whitespace", + "--3way", + "--check", + patch_file, + ] + p_check = subprocess.run(apply_check_args, cwd=args.llvm_dir) + + if p_check.returncode == 0: + # Patch will apply. + print(f"Applying {patch_file}...") + apply_args = [ + "git", + "apply", + "--ignore-whitespace", + "--3way", + patch_file, + ] + apply_args = subprocess.run(apply_args, cwd=args.llvm_dir, check=True) + applied_patches.append(patch_file) + else: + # Patch won't apply. + print(f"Unable to apply {patch_file}") + if args.restore_on_fail: + # Remove any patches that have already been applied. + while len(applied_patches) > 0: + r_patch = applied_patches.pop() + print(f"Reversing {r_patch}...") + reverse_args = [ + "git", + "apply", + "--ignore-whitespace", + "--3way", + "--reverse", + r_patch, + ] + p_check = subprocess.run( + reverse_args, cwd=args.llvm_dir, check=True + ) + print(f"Rollback successful, failure occured on {patch_file}") + sys.exit(2) + sys.exit(1) + print(f"All patches applied.") + + +main() diff --git a/patches/llvm-project-perf/0000-Placeholder-commit.patch b/patches/llvm-project-perf/0000-Placeholder-commit.patch new file mode 100644 index 00000000..ff8059cb --- /dev/null +++ b/patches/llvm-project-perf/0000-Placeholder-commit.patch @@ -0,0 +1,22 @@ +From e79697a54cba9bfc1a755ed048e42054d679de61 Mon Sep 17 00:00:00 2001 +From: David Candler +Date: Wed, 2 Oct 2024 14:13:31 +0100 +Subject: [PATCH] Placeholder commit + +--- + .gitignore | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/.gitignore b/.gitignore +index 0e7c6c790013..dfa0b8da0ccd 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -1,4 +1,4 @@ +-#==============================================================================# ++#==============================================================================# + # This file specifies intentionally untracked files that git should ignore. + # See: http://www.kernel.org/pub/software/scm/git/docs/gitignore.html + # +-- +2.34.1 + From 94fc59c534d9417a1f3f9c8cd692d7b0626dd7c6 Mon Sep 17 00:00:00 2001 From: David Candler Date: Fri, 4 Oct 2024 16:33:19 +0100 Subject: [PATCH 2/5] fixup! Add additional patch folder for performance patches --- cmake/patch_llvm.py | 76 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/cmake/patch_llvm.py b/cmake/patch_llvm.py index 4a9e2092..af8a868c 100644 --- a/cmake/patch_llvm.py +++ b/cmake/patch_llvm.py @@ -28,7 +28,7 @@ def main(): ) parser.add_argument( "--reset", - help="Clean and reset the repo to a specified commit before patching.", + help="Clean and hard reset the repo to a specified commit before patching.", ) parser.add_argument( "--restore_on_fail", @@ -37,29 +37,32 @@ def main(): ) args = parser.parse_args() + if args.llvm_dir: + git_cmd = ["git", "-C", args.llvm_dir] + else: + git_cmd = ["git"] + + abs_patch_dir = os.path.abspath(args.patchdir) + if args.reset: - reset_args = ["git", "reset", "--quiet", "--hard"] - subprocess.check_output(reset_args, cwd=args.llvm_dir) - clean_args = ["git", "clean", "--quiet", "--force", "-dx"] - subprocess.check_output(clean_args, cwd=args.llvm_dir) + reset_args = git_cmd + ["reset", "--quiet", "--hard", args.reset] + subprocess.check_output(reset_args) + clean_args = git_cmd + ["clean", "--quiet", "--force", "-dx", args.reset] + subprocess.check_output(clean_args) - patch_names = [] - for patch_name in os.listdir(args.patchdir): - if patch_name.endswith(".patch"): - patch_names.append(patch_name) + patch_names = [ + patch for patch in os.listdir(args.patchdir) if patch.endswith(".patch") + ] patch_names.sort() print(f"Found {len(patch_names)} patches to apply:") - for patch_name in patch_names: - print(patch_name) + print("\n".join(patch_names)) if args.method == "am": - merge_args = ["git", "am", "-k", "--ignore-whitespace", "--3way"] + merge_args = git_cmd + ["am", "-k", "--ignore-whitespace", "--3way"] for patch_name in patch_names: - merge_args.append(os.path.join(args.patchdir, patch_name)) - p = subprocess.run( - merge_args, cwd=args.llvm_dir, capture_output=True, text=True - ) + merge_args.append(os.path.join(abs_patch_dir, patch_name)) + p = subprocess.run(merge_args, capture_output=True, text=True) print(p.stdout) print(p.stderr) @@ -68,61 +71,58 @@ def main(): sys.exit(0) if args.restore_on_fail: # Check that the operation can be aborted. - if ( - 'To restore the original branch and stop patching, run "git am --abort".' - in p.stdout - ): + # git am does give any specific return codes, + # so check for unresolved working files. + if os.path.isdir(os.path.join(args.llvm_dir, ".git", "rebase-apply")): print("Aborting git am...") - subprocess.run(["git", "am", "--abort"], cwd=args.llvm_dir, check=True) + subprocess.run(git_cmd + ["am", "--abort"], check=True) + print(f"Abort successful.") sys.exit(2) + else: + print("Unable to abort.") sys.exit(1) else: applied_patches = [] for patch_name in patch_names: - patch_file = os.path.join(args.patchdir, patch_name) - print(f"Checking {patch_file}...") + patch_file = os.path.join(abs_patch_dir, patch_name) + print(f"Checking {patch_name}...") # Check that the patch applies before trying to apply it. - apply_check_args = [ - "git", + apply_check_args = git_cmd + [ "apply", "--ignore-whitespace", "--3way", "--check", patch_file, ] - p_check = subprocess.run(apply_check_args, cwd=args.llvm_dir) + p_check = subprocess.run(apply_check_args) if p_check.returncode == 0: # Patch will apply. - print(f"Applying {patch_file}...") - apply_args = [ - "git", + print(f"Applying {patch_name}...") + apply_args = git_cmd + [ "apply", "--ignore-whitespace", "--3way", patch_file, ] - apply_args = subprocess.run(apply_args, cwd=args.llvm_dir, check=True) - applied_patches.append(patch_file) + apply_args = subprocess.run(apply_args, check=True) + applied_patches.append(patch_name) else: # Patch won't apply. - print(f"Unable to apply {patch_file}") + print(f"Unable to apply {patch_name}") if args.restore_on_fail: # Remove any patches that have already been applied. while len(applied_patches) > 0: r_patch = applied_patches.pop() print(f"Reversing {r_patch}...") - reverse_args = [ - "git", + reverse_args = git_cmd + [ "apply", "--ignore-whitespace", "--3way", "--reverse", - r_patch, + os.path.join(abs_patch_dir, r_patch), ] - p_check = subprocess.run( - reverse_args, cwd=args.llvm_dir, check=True - ) + p_check = subprocess.run(reverse_args, check=True) print(f"Rollback successful, failure occured on {patch_file}") sys.exit(2) sys.exit(1) From c0d3dc711544f6956a3e7bc81ec73829f7e0ae44 Mon Sep 17 00:00:00 2001 From: David Candler Date: Tue, 8 Oct 2024 13:23:08 +0100 Subject: [PATCH 3/5] fixup! Add additional patch folder for performance patches --- docs/building-from-source.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/building-from-source.md b/docs/building-from-source.md index f62fac50..823d2ab7 100644 --- a/docs/building-from-source.md +++ b/docs/building-from-source.md @@ -148,3 +148,12 @@ The same build directory can be used for both native and MinGW toolchains. See [patches](https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/tree/main/patches) directory for the current set of differences from upstream. + +The patches for llvm-project are split between two folders, llvm-project and +llvm-project-perf. The former are generally required for building and +successfully running all tests. The patches in llvm-project-perf are optional, +and designed to improve performance in certain circumstances. + +To reduce divergence from upstream and potential patch conflicts, the +performance patches are not applied by default, but can be enabled for an +automatic checkout with the APPLY_LLVM_PERFORMANCE_PATCHES option. From 54b46512d4b8c156404fe0a26e6e9ab64b767c58 Mon Sep 17 00:00:00 2001 From: David Candler Date: Mon, 14 Oct 2024 15:42:30 +0100 Subject: [PATCH 4/5] fixup! Add additional patch folder for performance patches --- cmake/patch_llvm.py | 48 +++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/cmake/patch_llvm.py b/cmake/patch_llvm.py index af8a868c..073139f6 100644 --- a/cmake/patch_llvm.py +++ b/cmake/patch_llvm.py @@ -6,6 +6,7 @@ import argparse import os +import pathlib import subprocess import sys @@ -42,26 +43,23 @@ def main(): else: git_cmd = ["git"] - abs_patch_dir = os.path.abspath(args.patchdir) - if args.reset: reset_args = git_cmd + ["reset", "--quiet", "--hard", args.reset] subprocess.check_output(reset_args) clean_args = git_cmd + ["clean", "--quiet", "--force", "-dx", args.reset] subprocess.check_output(clean_args) - patch_names = [ - patch for patch in os.listdir(args.patchdir) if patch.endswith(".patch") - ] - patch_names.sort() + abs_patch_dir = os.path.abspath(args.patchdir) + patch_list = list(pathlib.Path(abs_patch_dir).glob("*.patch")) + patch_list.sort() - print(f"Found {len(patch_names)} patches to apply:") - print("\n".join(patch_names)) + print(f"Found {len(patch_list)} patches to apply:") + print("\n".join(p.name for p in patch_list)) if args.method == "am": merge_args = git_cmd + ["am", "-k", "--ignore-whitespace", "--3way"] - for patch_name in patch_names: - merge_args.append(os.path.join(abs_patch_dir, patch_name)) + for patch in patch_list: + merge_args.append(str(patch)) p = subprocess.run(merge_args, capture_output=True, text=True) print(p.stdout) print(p.stderr) @@ -73,7 +71,10 @@ def main(): # Check that the operation can be aborted. # git am does give any specific return codes, # so check for unresolved working files. - if os.path.isdir(os.path.join(args.llvm_dir, ".git", "rebase-apply")): + rebase_apply_path = os.path.join(".git", "rebase-apply") + if args.llvm_dir: + rebase_apply_path = os.path.join(args.llvm_dir, rebase_apply_path) + if os.path.isdir(rebase_apply_path): print("Aborting git am...") subprocess.run(git_cmd + ["am", "--abort"], check=True) print(f"Abort successful.") @@ -83,47 +84,48 @@ def main(): sys.exit(1) else: applied_patches = [] - for patch_name in patch_names: - patch_file = os.path.join(abs_patch_dir, patch_name) - print(f"Checking {patch_name}...") + for current_patch in patch_list: + print(f"Checking {current_patch.name}...") # Check that the patch applies before trying to apply it. apply_check_args = git_cmd + [ "apply", "--ignore-whitespace", "--3way", "--check", - patch_file, + str(current_patch), ] p_check = subprocess.run(apply_check_args) if p_check.returncode == 0: # Patch will apply. - print(f"Applying {patch_name}...") + print(f"Applying {current_patch.name}...") apply_args = git_cmd + [ "apply", "--ignore-whitespace", "--3way", - patch_file, + str(current_patch), ] apply_args = subprocess.run(apply_args, check=True) - applied_patches.append(patch_name) + applied_patches.append(current_patch) else: # Patch won't apply. - print(f"Unable to apply {patch_name}") + print(f"Unable to apply {current_patch.name}") if args.restore_on_fail: # Remove any patches that have already been applied. while len(applied_patches) > 0: - r_patch = applied_patches.pop() - print(f"Reversing {r_patch}...") + previous_patch = applied_patches.pop() + print(f"Reversing {previous_patch.name}...") reverse_args = git_cmd + [ "apply", "--ignore-whitespace", "--3way", "--reverse", - os.path.join(abs_patch_dir, r_patch), + str(previous_patch), ] p_check = subprocess.run(reverse_args, check=True) - print(f"Rollback successful, failure occured on {patch_file}") + print( + f"Rollback successful, failure occured on {current_patch.name}" + ) sys.exit(2) sys.exit(1) print(f"All patches applied.") From 713ec3b0fb81e4f1e7b89cc06848a99d1a196f1f Mon Sep 17 00:00:00 2001 From: David Candler Date: Mon, 14 Oct 2024 15:52:43 +0100 Subject: [PATCH 5/5] fixup! Add additional patch folder for performance patches --- cmake/patch_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/patch_llvm.py b/cmake/patch_llvm.py index 073139f6..85bf535b 100644 --- a/cmake/patch_llvm.py +++ b/cmake/patch_llvm.py @@ -69,7 +69,7 @@ def main(): sys.exit(0) if args.restore_on_fail: # Check that the operation can be aborted. - # git am does give any specific return codes, + # git am doesn't give any specific return codes, # so check for unresolved working files. rebase_apply_path = os.path.join(".git", "rebase-apply") if args.llvm_dir: