From 5743e3fd8153e0f25c2bfbbc1ac4af84e0678233 Mon Sep 17 00:00:00 2001 From: bbhtt Date: Thu, 4 Sep 2025 09:36:42 +0530 Subject: [PATCH 1/2] builder-git: Add support for fetching and checking out Git LFS objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This explicitly fetches Git LFS objects as long as git-lfs is available on host (but doesn't need to be configured) Fixes https://github.com/flatpak/flatpak-builder/issues/463 Co-authored-by: Bartłomiej Piotrowski --- .github/workflows/check.yml | 7 +-- README.md | 1 + src/builder-git.c | 99 ++++++++++++++++++++++++++++++++++++- 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 1990c2f5..35907f55 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -26,7 +26,7 @@ jobs: libseccomp-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \ libgirepository1.0-dev libappstream-dev libdconf-dev clang socat flatpak \ libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch libarchive-tools \ - docbook-xsl xmlto xsltproc + docbook-xsl xmlto xsltproc git-lfs - name: Check out flatpak uses: actions/checkout@v4 with: @@ -59,7 +59,7 @@ jobs: libseccomp-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \ libgirepository1.0-dev libappstream-dev libdconf-dev clang flatpak \ libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch libarchive-tools \ - docbook-xsl xmlto xsltproc + docbook-xsl xmlto xsltproc git-lfs - name: Check out flatpak uses: actions/checkout@v4 with: @@ -135,7 +135,8 @@ jobs: libarchive-tools \ docbook-xsl \ xmlto \ - xsltproc + xsltproc \ + git-lfs - name: Check out flatpak-builder uses: actions/checkout@v4 diff --git a/README.md b/README.md index 852cfc7c..10f3ec1d 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Very commonly used: * git * 7z * bsdunzip (libarchive) + * git-lfs Rarely used: diff --git a/src/builder-git.c b/src/builder-git.c index 64b48eb8..a9ee6e29 100644 --- a/src/builder-git.c +++ b/src/builder-git.c @@ -148,6 +148,85 @@ git_has_version (int major, return TRUE; } +static gboolean +git_lfs (GFile *dir, + char **output, + GSubprocessFlags flags, + GError **error, + const char **args) +{ + g_autoptr(GPtrArray) full_args = g_ptr_array_new (); + gboolean res; + + g_ptr_array_add (full_args, "git"); + g_ptr_array_add (full_args, "lfs"); + + for (const char **arg = args; *arg; arg++) + g_ptr_array_add (full_args, (char *) *arg); + + g_ptr_array_add (full_args, NULL); + + res = flatpak_spawnv (dir, output, flags, error, (const char **) full_args->pdata, NULL); + + return res; +} + +static gboolean +git_lfs_is_available (void) +{ + static gsize lfs_available = 0; + + if (g_once_init_enter (&lfs_available)) + { + g_autoptr(GError) my_error = NULL; + gsize new_lfs_available = 2; + + if (git_lfs (NULL, NULL, 0, &my_error, (const char *[]){"version", NULL})) + new_lfs_available = 1; + else + g_debug ("git-lfs is not available: %s", my_error->message); + + g_once_init_leave (&lfs_available, new_lfs_available); + } + + return lfs_available == 1; +} + +static gboolean +builder_git_run_lfs (GFile *dir, + GError **error, + const char *subcommand, + ...) +{ + va_list ap; + g_autoptr(GPtrArray) args = g_ptr_array_new (); + const char *arg; + gboolean ok; + + if (!git_lfs_is_available ()) + return TRUE; + + g_ptr_array_add (args, (char *) subcommand); + + va_start (ap, subcommand); + while ((arg = va_arg (ap, const char *))) + g_ptr_array_add (args, (char *) arg); + va_end (ap); + + g_ptr_array_add (args, NULL); + + g_print ("Running git lfs %s\n", subcommand); + ok = git_lfs (dir, NULL, 0, error, (const char **) args->pdata); + + if (!ok) + { + git_lfs (dir, NULL, 0, NULL, (const char *[]){"logs", "last", NULL}); + return FALSE; + } + + return TRUE; +} + static gboolean git_version_supports_fsck_and_shallow (void) { @@ -578,6 +657,10 @@ builder_git_mirror_repo (const char *repo_location, origin, full_ref_mapping, NULL)) return FALSE; + if (!builder_git_run_lfs (mirror_dir, error, + "fetch", "--all", NULL)) + return FALSE; + /* It turns out that older versions of git (at least 2.7.4) * cannot check out a commit unless a real tag/branch points * to it, which is not the case for e.g. gitbug pull requests. @@ -606,6 +689,10 @@ builder_git_mirror_repo (const char *repo_location, was_shallow ? "--unshallow" : NULL, NULL)) return FALSE; + + if (!builder_git_run_lfs (mirror_dir, error, + "fetch", "--all", NULL)) + return FALSE; } if (alternates) @@ -708,6 +795,10 @@ builder_git_shallow_mirror_ref (const char *repo_location, "fetch", "--depth", "1", "origin", full_ref_colon_full_ref, NULL)) return FALSE; + if (!builder_git_run_lfs (mirror_dir, error, + "fetch", "origin", full_ref, NULL)) + return FALSE; + /* Always mirror submodules */ current_commit = git_get_current_commit (mirror_dir, ref, FALSE, context, error); if (current_commit == NULL) @@ -858,10 +949,16 @@ builder_git_checkout (const char *repo_location, "config", "--bool", "core.bare", "false", NULL)) return FALSE; + if (!builder_git_run_lfs (dest, error, "install", "--local", NULL)) + return FALSE; + if (!git (dest, NULL, 0, error, "checkout", branch, NULL)) return FALSE; + if (!builder_git_run_lfs (dest, error, "checkout", NULL)) + return FALSE; + if (mirror_flags & FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES) if (!git_extract_submodule (repo_location, dest, branch, context, error)) return FALSE; @@ -897,4 +994,4 @@ builder_git_get_default_branch (const char *repo_location) g_debug ("Failed to auto-detect default branch from git output"); return g_strdup ("master"); -} \ No newline at end of file +} From a873aacd184cdce4d23ee66d4e1d4b04d8137f5d Mon Sep 17 00:00:00 2001 From: bbhtt Date: Thu, 4 Sep 2025 09:52:43 +0530 Subject: [PATCH 2/2] builder-git: Add a property to disable explicitly fetching LFS The previous commit will attempt to fetch LFS by default for Git sources even if LFS is not configured (but available). This allows to disable that behaviour on a per source basis since repos may have huge LFS assets which may be undesirable to fetch. --- data/flatpak-manifest.schema.json | 4 ++++ doc/flatpak-manifest.xml | 4 ++++ src/builder-git.c | 18 ++++++++++++------ src/builder-git.h | 1 + src/builder-source-git.c | 26 ++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/data/flatpak-manifest.schema.json b/data/flatpak-manifest.schema.json index 6172c1fd..b9bb8be4 100644 --- a/data/flatpak-manifest.schema.json +++ b/data/flatpak-manifest.schema.json @@ -840,6 +840,10 @@ "disable-submodules": { "description": "Don't checkout the git submodules when cloning the repository.", "type": "boolean" + }, + "disable-lfs": { + "description": "Don't explicitly fetch or checkout LFS git objects.", + "type": "boolean" } }, "patternProperties": { diff --git a/doc/flatpak-manifest.xml b/doc/flatpak-manifest.xml index f0536636..1bd03591 100644 --- a/doc/flatpak-manifest.xml +++ b/doc/flatpak-manifest.xml @@ -772,6 +772,10 @@ (boolean) Don't checkout the git submodules when cloning the repository. + + (boolean) + Don't explicitly fetch or checkout LFS git objects. This will be ignored by Git if LFS filters are active in system or global gitconfig. + diff --git a/src/builder-git.c b/src/builder-git.c index a9ee6e29..b1f14063 100644 --- a/src/builder-git.c +++ b/src/builder-git.c @@ -194,6 +194,7 @@ git_lfs_is_available (void) static gboolean builder_git_run_lfs (GFile *dir, + FlatpakGitMirrorFlags flags, GError **error, const char *subcommand, ...) @@ -203,6 +204,9 @@ builder_git_run_lfs (GFile *dir, const char *arg; gboolean ok; + if (flags & FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS) + return TRUE; + if (!git_lfs_is_available ()) return TRUE; @@ -657,7 +661,7 @@ builder_git_mirror_repo (const char *repo_location, origin, full_ref_mapping, NULL)) return FALSE; - if (!builder_git_run_lfs (mirror_dir, error, + if (!builder_git_run_lfs (mirror_dir, flags, error, "fetch", "--all", NULL)) return FALSE; @@ -690,7 +694,7 @@ builder_git_mirror_repo (const char *repo_location, NULL)) return FALSE; - if (!builder_git_run_lfs (mirror_dir, error, + if (!builder_git_run_lfs (mirror_dir, flags, error, "fetch", "--all", NULL)) return FALSE; } @@ -795,7 +799,7 @@ builder_git_shallow_mirror_ref (const char *repo_location, "fetch", "--depth", "1", "origin", full_ref_colon_full_ref, NULL)) return FALSE; - if (!builder_git_run_lfs (mirror_dir, error, + if (!builder_git_run_lfs (mirror_dir, flags, error, "fetch", "origin", full_ref, NULL)) return FALSE; @@ -949,14 +953,16 @@ builder_git_checkout (const char *repo_location, "config", "--bool", "core.bare", "false", NULL)) return FALSE; - if (!builder_git_run_lfs (dest, error, "install", "--local", NULL)) - return FALSE; + if (!builder_git_run_lfs (dest, mirror_flags, error, + "install", "--local", NULL)) + return FALSE; if (!git (dest, NULL, 0, error, "checkout", branch, NULL)) return FALSE; - if (!builder_git_run_lfs (dest, error, "checkout", NULL)) + if (!builder_git_run_lfs (dest, mirror_flags, error, + "checkout", NULL)) return FALSE; if (mirror_flags & FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES) diff --git a/src/builder-git.h b/src/builder-git.h index d27fe5e4..db297795 100644 --- a/src/builder-git.h +++ b/src/builder-git.h @@ -31,6 +31,7 @@ typedef enum { FLATPAK_GIT_MIRROR_FLAGS_DISABLE_FSCK = 1 << 2, FLATPAK_GIT_MIRROR_FLAGS_DISABLE_SHALLOW = 1 << 3, FLATPAK_GIT_MIRROR_FLAGS_WILL_FETCH_FROM = 1 << 4, + FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS = 1 << 5, } FlatpakGitMirrorFlags; gboolean builder_git_mirror_repo (const char *repo_location, diff --git a/src/builder-source-git.c b/src/builder-source-git.c index e6b9ad9f..cf26ff9b 100644 --- a/src/builder-source-git.c +++ b/src/builder-source-git.c @@ -48,6 +48,7 @@ struct BuilderSourceGit gboolean disable_fsckobjects; gboolean disable_shallow_clone; gboolean disable_submodules; + gboolean disable_lfs; }; typedef struct @@ -67,6 +68,7 @@ enum { PROP_DISABLE_FSCKOBJECTS, PROP_DISABLE_SHALLOW_CLONE, PROP_DISABLE_SUBMODULES, + PROP_DISABLE_LFS, LAST_PROP }; @@ -128,6 +130,10 @@ builder_source_git_get_property (GObject *object, g_value_set_boolean (value, self->disable_submodules); break; + case PROP_DISABLE_LFS: + g_value_set_boolean (value, self->disable_lfs); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); } @@ -180,6 +186,10 @@ builder_source_git_set_property (GObject *object, self->disable_submodules = g_value_get_boolean (value); break; + case PROP_DISABLE_LFS: + self->disable_lfs = g_value_get_boolean (value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); } @@ -255,6 +265,8 @@ builder_source_git_download (BuilderSource *source, if (!self->disable_submodules) flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES; + if (self->disable_lfs) + flags |= FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS; if (update_vcs) flags |= FLATPAK_GIT_MIRROR_FLAGS_UPDATE; if (self->disable_fsckobjects) @@ -303,6 +315,9 @@ builder_source_git_extract (BuilderSource *source, if (!self->disable_submodules) mirror_flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES; + if (self->disable_lfs) + mirror_flags |= FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS; + if (!builder_git_checkout (location, get_branch (self, location), dest, context, mirror_flags, error)) return FALSE; @@ -336,6 +351,9 @@ builder_source_git_bundle (BuilderSource *source, if (!self->disable_submodules) flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES; + if (self->disable_lfs) + flags |= FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS; + if (!builder_git_shallow_mirror_ref (location, flatpak_file_get_path_cached (mirror_dir), flags, @@ -481,6 +499,14 @@ builder_source_git_class_init (BuilderSourceGitClass *klass) "", FALSE, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, + PROP_DISABLE_LFS, + g_param_spec_boolean ("disable-lfs", + "", + "", + FALSE, + G_PARAM_READWRITE)); } static void