From a064b0be55f60b0b8281b4a6a38917eab4b9c50f Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Wed, 23 Apr 2025 10:15:34 +0200 Subject: [PATCH 1/7] ci/github: install git before checking out the repository The GitHub's CI workflow uses 'actions/checkout@v4' to checkout the repository. This action defaults to using the GitHub REST API to obtain the repository if the `git` executable isn't available. The step to build Git in the GitHub workflow can be summarized as: ... - uses: actions/checkout@v4 #1 - run: ci/install-dependencies.sh #2 ... - run: sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh #3 ... Step #1, clones the repository, since the `git` executable isn't present at this step, it uses GitHub's REST API to obtain a tar of the repository. Step #2, installs all dependencies, which includes the `git` executable. Step #3, sets up the build, which includes setting up meson in the meson job. At this point the `git` executable is present. This means while the `git` executable is present, the repository doesn't contain the '.git' folder. To keep both the CI's (GitLab and GitHub) behavior consistent and to ensure that the build is performed on a real-world scenario, install `git` before the repository is checked out. This ensures that 'actions/checkout@v4' will clone the repository instead of using a tarball. We also update the package cache while installing `git`, this is because some distros will fail to locate the package without updating the cache. Helped-by: Phillip Wood Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37541f3d10daba..e9112b3a64cd52 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -414,6 +414,20 @@ jobs: - name: prepare libc6 for actions if: matrix.vector.jobname == 'linux32' run: apt -q update && apt -q -y install libc6-amd64 lib64stdc++6 + - name: install git in container + run: | + if command -v git + then + : # nothing to do + elif command -v apk + then + apk add --update git + elif command -v dnf + then + dnf -yq update && dnf -yq install git + else + apt-get -q update && apt-get -q -y install git + fi - uses: actions/checkout@v4 - run: ci/install-dependencies.sh - run: useradd builder --create-home From 8e980b7f2501c4b9429394f7287757af7aef81ba Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Wed, 23 Apr 2025 10:15:35 +0200 Subject: [PATCH 2/7] coccinelle: meson: rename variables to be more specific In Meson, included subdirs export their variables to top level Meson builds. In 'contrib/coccinelle/meson.build', we define two such variables `sources` and `headers`. While these variables are specific to the checks in the 'contrib/coccinelle/' directory, they also pollute the top level 'meson.build'. Rename them to be more specific, this ensures that they aren't mistakenly used in the upper levels and avoid variable name collisions. While here, change the empty list denotation to be consistent with other places. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- contrib/coccinelle/meson.build | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/coccinelle/meson.build b/contrib/coccinelle/meson.build index ea054c924f400f..03ce52d7522b76 100644 --- a/contrib/coccinelle/meson.build +++ b/contrib/coccinelle/meson.build @@ -55,18 +55,18 @@ concatenated_rules = custom_target( capture: true, ) -sources = [ ] +coccinelle_sources = [] foreach source : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.c', third_party_sources, check: true).stdout().split() - sources += source + coccinelle_sources += source endforeach -headers = [ ] +coccinelle_headers = [] foreach header : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.h', third_party_sources, check: true).stdout().split() - headers += meson.project_source_root() / header + coccinelle_headers += meson.project_source_root() / header endforeach patches = [ ] -foreach source : sources +foreach source : coccinelle_sources patches += custom_target( command: [ spatch, @@ -78,7 +78,7 @@ foreach source : sources input: meson.project_source_root() / source, output: source.underscorify() + '.patch', capture: true, - depend_files: headers, + depend_files: coccinelle_headers, ) endforeach From 1597b6e86e3c13438a0e469b10048e73039e8023 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Wed, 23 Apr 2025 10:15:36 +0200 Subject: [PATCH 3/7] meson: move headers definition from 'contrib/coccinelle' The Meson build for coccinelle static analysis lists all headers to analyse. Due to the way Meson exports variables between subdirs, this variable is also available in the root Meson build. An upcoming commit, will add a new check complimenting 'hdr-check' in the Makefile. This would require the list of headers. So move the 'coccinelle_headers' to the root Meson build and rename it to 'headers', remove the root path being appended to each header and retain that in the coccinelle Meson build since it is specific to the coccinelle build. Also move the 'third_party_sources' variable to the root Meson build since it is also a dependency for the 'headers' variable. This also makes it easier to understand as the variable is now propagated from the top level to the bottom. While 'headers_to_check' is only computed when we have a repository and the 'git' executable is present, the variable itself is exposed as an empty array. This allows dependencies in upcoming commits to simply check for length of the array and not worry about dependencies required to actually populate the array. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- contrib/coccinelle/meson.build | 17 +---------------- meson.build | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/contrib/coccinelle/meson.build b/contrib/coccinelle/meson.build index 03ce52d7522b76..4f07824402f317 100644 --- a/contrib/coccinelle/meson.build +++ b/contrib/coccinelle/meson.build @@ -8,21 +8,6 @@ if not spatch.found() subdir_done() endif -third_party_sources = [ - ':!contrib', - ':!compat/inet_ntop.c', - ':!compat/inet_pton.c', - ':!compat/nedmalloc', - ':!compat/obstack.*', - ':!compat/poll', - ':!compat/regex', - ':!sha1collisiondetection', - ':!sha1dc', - ':!t/unit-tests/clar', - ':!t/unit-tests/clar', - ':!t/t[0-9][0-9][0-9][0-9]*', -] - rules = [ 'array.cocci', 'commit.cocci', @@ -61,7 +46,7 @@ foreach source : run_command(git, '-C', meson.project_source_root(), 'ls-files', endforeach coccinelle_headers = [] -foreach header : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.h', third_party_sources, check: true).stdout().split() +foreach header : headers_to_check coccinelle_headers += meson.project_source_root() / header endforeach diff --git a/meson.build b/meson.build index e98cfa4909f288..e147ddff286bec 100644 --- a/meson.build +++ b/meson.build @@ -633,6 +633,28 @@ builtin_sources = [ 'builtin/write-tree.c', ] +third_party_sources = [ + ':!contrib', + ':!compat/inet_ntop.c', + ':!compat/inet_pton.c', + ':!compat/nedmalloc', + ':!compat/obstack.*', + ':!compat/poll', + ':!compat/regex', + ':!sha1collisiondetection', + ':!sha1dc', + ':!t/unit-tests/clar', + ':!t/unit-tests/clar', + ':!t/t[0-9][0-9][0-9][0-9]*', +] + +headers_to_check = [] +if git.found() and fs.exists(meson.project_source_root() / '.git') + foreach header : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.h', third_party_sources, check: true).stdout().split() + headers_to_check += header + endforeach +endif + if not get_option('breaking_changes') builtin_sources += 'builtin/pack-redundant.c' endif From 7e873eb390204dbe55ec4101fb7ab737f5ae0bc6 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Wed, 23 Apr 2025 10:15:37 +0200 Subject: [PATCH 4/7] meson: rename 'third_party_sources' to 'third_party_excludes' The 'third_party_sources' variable was moved to the root 'meson.build' file in the previous commit. The variable is actually used to exclude third party sources, so rename it accordingly to 'third_party_excludes' to avoid confusion. While here, remove a duplicate from the list. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- contrib/coccinelle/meson.build | 2 +- meson.build | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/coccinelle/meson.build b/contrib/coccinelle/meson.build index 4f07824402f317..dc3f73c2e7b117 100644 --- a/contrib/coccinelle/meson.build +++ b/contrib/coccinelle/meson.build @@ -41,7 +41,7 @@ concatenated_rules = custom_target( ) coccinelle_sources = [] -foreach source : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.c', third_party_sources, check: true).stdout().split() +foreach source : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.c', third_party_excludes, check: true).stdout().split() coccinelle_sources += source endforeach diff --git a/meson.build b/meson.build index e147ddff286bec..4618804c7a19b1 100644 --- a/meson.build +++ b/meson.build @@ -633,7 +633,7 @@ builtin_sources = [ 'builtin/write-tree.c', ] -third_party_sources = [ +third_party_excludes = [ ':!contrib', ':!compat/inet_ntop.c', ':!compat/inet_pton.c', @@ -644,13 +644,12 @@ third_party_sources = [ ':!sha1collisiondetection', ':!sha1dc', ':!t/unit-tests/clar', - ':!t/unit-tests/clar', ':!t/t[0-9][0-9][0-9][0-9]*', ] headers_to_check = [] if git.found() and fs.exists(meson.project_source_root() / '.git') - foreach header : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.h', third_party_sources, check: true).stdout().split() + foreach header : run_command(git, '-C', meson.project_source_root(), 'ls-files', '--deduplicate', '*.h', third_party_excludes, check: true).stdout().split() headers_to_check += header endforeach endif From 02a132616ac69cd546c823c47afd60eab738792c Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Wed, 23 Apr 2025 10:15:38 +0200 Subject: [PATCH 5/7] meson: add support for 'hdr-check' The Makefile supports a target called 'hdr-check', which checks if individual header files can be independently compiled. Let's port this functionality to Meson, our new build system too. The implementation resembles that of the Makefile and provides the same check. Since meson builds are out-of-tree, header dependencies are not automatically met. So unlike the Makefile version, we also need to add the required dependencies. Also add the 'xdiff/' dir to the list of 'third_party_sources' as those headers must be skipped from the checks too. This also skips the folder from the 'coccinelle' checks, this is okay, since this code is an external dependency. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- meson.build | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/meson.build b/meson.build index 4618804c7a19b1..22fc65ec80ba6a 100644 --- a/meson.build +++ b/meson.build @@ -645,6 +645,7 @@ third_party_excludes = [ ':!sha1dc', ':!t/unit-tests/clar', ':!t/t[0-9][0-9][0-9][0-9]*', + ':!xdiff', ] headers_to_check = [] @@ -1994,6 +1995,68 @@ endif subdir('contrib') +exclude_from_check_headers = [ + 'compat/', + 'unicode-width.h', +] + +if sha1_backend != 'openssl' + exclude_from_check_headers += 'sha1/openssl.h' +endif +if sha256_backend != 'openssl' + exclude_from_check_headers += 'sha256/openssl.h' +endif +if sha256_backend != 'nettle' + exclude_from_check_headers += 'sha256/nettle.h' +endif +if sha256_backend != 'gcrypt' + exclude_from_check_headers += 'sha256/gcrypt.h' +endif + +if headers_to_check.length() != 0 and compiler.get_argument_syntax() == 'gcc' + hco_targets = [] + foreach h : headers_to_check + skip_header = false + foreach exclude : exclude_from_check_headers + if h.startswith(exclude) + skip_header = true + break + endif + endforeach + + if skip_header + continue + endif + + hcc = custom_target( + input: h, + output: h.underscorify() + 'cc', + command: [ + shell, + '-c', + 'echo \'#include "git-compat-util.h"\' > @OUTPUT@ && echo \'#include "' + h + '"\' >> @OUTPUT@' + ] + ) + + hco = custom_target( + input: hcc, + output: fs.replace_suffix(h.underscorify(), '.hco'), + command: [ + compiler.cmd_array(), + libgit_c_args, + '-I', meson.project_source_root(), + '-I', meson.project_source_root() / 't/unit-tests', + '-o', '/dev/null', + '-c', '-xc', + '@INPUT@' + ] + ) + hco_targets += hco + endforeach + + alias_target('hdr-check', hco_targets) +endif + foreach key, value : { 'DIFF': diff.full_path(), 'GIT_SOURCE_DIR': meson.project_source_root(), From 04a13ed8a7c07ef383f19adc16494ec481960bdb Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Wed, 23 Apr 2025 10:15:39 +0200 Subject: [PATCH 6/7] makefile/meson: add 'check-headers' as alias for 'hdr-check' The 'hdr-check' target in Meson and makefile is used to check if headers can be compiled individually. The naming however isn't readable as 'hdr' is not a common shortforme for 'header', neither is it an abbreviation. Let's introduce 'check-headers' as an alternative target for 'hdr-check' and add a `TODO` to deprecate the latter after 2 releases. Since this is an internal tool, we can use a shorter deprecation cycle. Change existing usage of 'hdr-check' in 'ci/run-static-analysis.sh' to also use 'check-headers'. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- Makefile | 4 +++- ci/run-static-analysis.sh | 2 +- meson.build | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ac32d2d0bdae93..961ee508be37c4 100644 --- a/Makefile +++ b/Makefile @@ -3326,8 +3326,10 @@ HCC = $(HCO:hco=hcc) $(HCO): %.hco: %.hcc $(GENERATED_H) FORCE $(QUIET_HDR)$(CC) $(ALL_CFLAGS) -o /dev/null -c -xc $< -.PHONY: hdr-check $(HCO) +# TODO: deprecate 'hdr-check' in lieu of 'check-headers' in Git 2.51+ +.PHONY: hdr-check check-headers $(HCO) hdr-check: $(HCO) +check-headers: hdr-check .PHONY: style style: diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh index 0d51e5ce0e7cb4..60c175a094331d 100755 --- a/ci/run-static-analysis.sh +++ b/ci/run-static-analysis.sh @@ -26,7 +26,7 @@ then exit 1 fi -make hdr-check || +make check-headers || exit 1 make check-pot diff --git a/meson.build b/meson.build index 22fc65ec80ba6a..569e3888fb2d0b 100644 --- a/meson.build +++ b/meson.build @@ -2054,7 +2054,9 @@ if headers_to_check.length() != 0 and compiler.get_argument_syntax() == 'gcc' hco_targets += hco endforeach - alias_target('hdr-check', hco_targets) + # TODO: deprecate 'hdr-check' in lieu of 'check-headers' in Git 2.51+ + hdr_check = alias_target('hdr-check', hco_targets) + alias_target('check-headers', hdr_check) endif foreach key, value : { From 4183edd2135fbf85f6c98f8d2608ae7837c5517c Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 23 Apr 2025 13:28:17 -0700 Subject: [PATCH 7/7] citweak: for now disable jgit download --- ci/install-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index be9ba5e30a477b..7cf62ea9681745 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -74,8 +74,8 @@ ubuntu-*|i386/ubuntu-*|debian-*) -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" - chmod a+x "$CUSTOM_PATH/jgit" + : wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" + : chmod a+x "$CUSTOM_PATH/jgit" ;; esac ;;