builder-git: Checkout LFS objects

.github/workflows/check.yml

@@ -26,7 +26,7 @@ jobs:
         libseccomp-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \
         libgirepository1.0-dev libappstream-dev libdconf-dev clang socat flatpak \
         libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch libarchive-tools \
-        docbook-xsl xmlto xsltproc
+        docbook-xsl xmlto xsltproc git-lfs
     - name: Check out flatpak
       uses: actions/checkout@v4
       with:
@@ -59,7 +59,7 @@ jobs:
         libseccomp-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \
         libgirepository1.0-dev libappstream-dev libdconf-dev clang flatpak \
         libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch libarchive-tools \
-        docbook-xsl xmlto xsltproc
+        docbook-xsl xmlto xsltproc git-lfs
     - name: Check out flatpak
       uses: actions/checkout@v4
       with:
@@ -135,7 +135,8 @@ jobs:
           libarchive-tools \
           docbook-xsl \
           xmlto \
-          xsltproc
+          xsltproc \
+          git-lfs
 
     - name: Check out flatpak-builder
       uses: actions/checkout@v4

README.md

@@ -41,6 +41,7 @@ Very commonly used:
  * git
  * 7z
  * bsdunzip (libarchive)
+ * git-lfs
 
 Rarely used:
 

data/flatpak-manifest.schema.json

@@ -840,6 +840,10 @@
         "disable-submodules": {
           "description": "Don't checkout the git submodules when cloning the repository.",
           "type": "boolean"
+        },
+        "disable-lfs": {
+          "description": "Don't explicitly fetch or checkout LFS git objects.",
+          "type": "boolean"
         }
       },
       "patternProperties": {

doc/flatpak-manifest.xml

@@ -772,6 +772,10 @@
                         <term><option>disable-submodules</option> (boolean)</term>
                         <listitem><para>Don't checkout the git submodules when cloning the repository.</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>disable-lfs</option> (boolean)</term>
+                        <listitem><para>Don't explicitly fetch or checkout LFS git objects. This will be ignored by Git if LFS filters are active in system or global gitconfig.</para></listitem>
+                    </varlistentry>
                 </variablelist>
             </refsect3>
             <refsect3>

src/builder-git.c

@@ -148,6 +148,89 @@ git_has_version (int major,
   return TRUE;
 }
 
+static gboolean
+git_lfs (GFile   *dir,
+         char   **output,
+         GSubprocessFlags flags,
+         GError **error,
+         const char **args)
+{
+  g_autoptr(GPtrArray) full_args = g_ptr_array_new ();
+  gboolean res;
+
+  g_ptr_array_add (full_args, "git");
+  g_ptr_array_add (full_args, "lfs");
+
+  for (const char **arg = args; *arg; arg++)
+    g_ptr_array_add (full_args, (char *) *arg);
+
+  g_ptr_array_add (full_args, NULL);
+
+  res = flatpak_spawnv (dir, output, flags, error, (const char **) full_args->pdata, NULL);
+
+  return res;
+}
+
+static gboolean
+git_lfs_is_available (void)
+{
+  static gsize lfs_available = 0;
+
+  if (g_once_init_enter (&lfs_available))
+    {
+      g_autoptr(GError) my_error = NULL;
+      gsize new_lfs_available = 2;
+
+      if (git_lfs (NULL, NULL, 0, &my_error, (const char *[]){"version", NULL}))
+        new_lfs_available = 1;
+      else
+        g_debug ("git-lfs is not available: %s", my_error->message);
+
+      g_once_init_leave (&lfs_available, new_lfs_available);
+    }
+
+  return lfs_available == 1;
+}
+
+static gboolean
+builder_git_run_lfs (GFile       *dir,
+                     FlatpakGitMirrorFlags flags,
+                     GError     **error,
+                     const char  *subcommand,
+                     ...)
+{
+  va_list ap;
+  g_autoptr(GPtrArray) args = g_ptr_array_new ();
+  const char *arg;
+  gboolean ok;
+
+  if (flags & FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS)
+    return TRUE;
+
+  if (!git_lfs_is_available ())
+    return TRUE;
+
+  g_ptr_array_add (args, (char *) subcommand);
+
+  va_start (ap, subcommand);
+  while ((arg = va_arg (ap, const char *)))
+    g_ptr_array_add (args, (char *) arg);
+  va_end (ap);
+
+  g_ptr_array_add (args, NULL);
+
+  g_print ("Running git lfs %s\n", subcommand);
+  ok = git_lfs (dir, NULL, 0, error, (const char **) args->pdata);
+
+  if (!ok)
+    {
+      git_lfs (dir, NULL, 0, NULL, (const char *[]){"logs", "last", NULL});
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
 static gboolean
 git_version_supports_fsck_and_shallow (void)
 {
@@ -578,6 +661,10 @@ builder_git_mirror_repo (const char     *repo_location,
                     origin, full_ref_mapping, NULL))
             return FALSE;
 
+          if (!builder_git_run_lfs (mirror_dir, flags, error,
+                                    "fetch", "--all", NULL))
+            return FALSE;
+
 	  /* It turns out that older versions of git (at least 2.7.4)
 	   * cannot check out a commit unless a real tag/branch points
 	   * to it, which is not the case for e.g. gitbug pull requests.
@@ -606,6 +693,10 @@ builder_git_mirror_repo (const char     *repo_location,
                     was_shallow ? "--unshallow" : NULL,
                     NULL))
             return FALSE;
+
+          if (!builder_git_run_lfs (mirror_dir, flags, error,
+                                    "fetch", "--all", NULL))
+            return FALSE;
         }
 
       if (alternates)
@@ -708,6 +799,10 @@ builder_git_shallow_mirror_ref (const char     *repo_location,
             "fetch", "--depth", "1", "origin", full_ref_colon_full_ref, NULL))
     return FALSE;
 
+  if (!builder_git_run_lfs (mirror_dir, flags, error,
+                            "fetch", "origin", full_ref, NULL))
+    return FALSE;
+
   /* Always mirror submodules */
   current_commit = git_get_current_commit (mirror_dir, ref, FALSE, context, error);
   if (current_commit == NULL)
@@ -858,10 +953,18 @@ builder_git_checkout (const char     *repo_location,
             "config", "--bool", "core.bare", "false", NULL))
     return FALSE;
 
+  if (!builder_git_run_lfs (dest, mirror_flags, error,
+                            "install", "--local", NULL))
+      return FALSE;
+
   if (!git (dest, NULL, 0, error,
             "checkout", branch, NULL))
     return FALSE;
 
+  if (!builder_git_run_lfs (dest, mirror_flags, error,
+                            "checkout", NULL))
+    return FALSE;
+
   if (mirror_flags & FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES)
     if (!git_extract_submodule (repo_location, dest, branch, context, error))
       return FALSE;
@@ -897,4 +1000,4 @@ builder_git_get_default_branch (const char *repo_location)
 
   g_debug ("Failed to auto-detect default branch from git output");
   return g_strdup ("master");
-}
+}

src/builder-git.h

@@ -31,6 +31,7 @@ typedef enum {
   FLATPAK_GIT_MIRROR_FLAGS_DISABLE_FSCK = 1 << 2,
   FLATPAK_GIT_MIRROR_FLAGS_DISABLE_SHALLOW = 1 << 3,
   FLATPAK_GIT_MIRROR_FLAGS_WILL_FETCH_FROM = 1 << 4,
+  FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS = 1 << 5,
 } FlatpakGitMirrorFlags;
 
 gboolean builder_git_mirror_repo        (const char      *repo_location,

src/builder-source-git.c

@@ -48,6 +48,7 @@ struct BuilderSourceGit
   gboolean      disable_fsckobjects;
   gboolean      disable_shallow_clone;
   gboolean      disable_submodules;
+  gboolean      disable_lfs;
 };
 
 typedef struct
@@ -67,6 +68,7 @@ enum {
   PROP_DISABLE_FSCKOBJECTS,
   PROP_DISABLE_SHALLOW_CLONE,
   PROP_DISABLE_SUBMODULES,
+  PROP_DISABLE_LFS,
   LAST_PROP
 };
 
@@ -128,6 +130,10 @@ builder_source_git_get_property (GObject    *object,
       g_value_set_boolean (value, self->disable_submodules);
       break;
 
+    case PROP_DISABLE_LFS:
+      g_value_set_boolean (value, self->disable_lfs);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -180,6 +186,10 @@ builder_source_git_set_property (GObject      *object,
       self->disable_submodules = g_value_get_boolean (value);
       break;
 
+    case PROP_DISABLE_LFS:
+      self->disable_lfs = g_value_get_boolean (value);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -255,6 +265,8 @@ builder_source_git_download (BuilderSource  *source,
 
   if (!self->disable_submodules)
     flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES;
+  if (self->disable_lfs)
+    flags |= FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS;
   if (update_vcs)
     flags |= FLATPAK_GIT_MIRROR_FLAGS_UPDATE;
   if (self->disable_fsckobjects)
@@ -303,6 +315,9 @@ builder_source_git_extract (BuilderSource  *source,
   if (!self->disable_submodules)
     mirror_flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES;
 
+  if (self->disable_lfs)
+    mirror_flags |= FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS;
+
   if (!builder_git_checkout (location, get_branch (self, location),
                              dest, context, mirror_flags, error))
     return FALSE;
@@ -336,6 +351,9 @@ builder_source_git_bundle (BuilderSource  *source,
   if (!self->disable_submodules)
     flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES;
 
+  if (self->disable_lfs)
+    flags |= FLATPAK_GIT_MIRROR_FLAGS_DISABLE_LFS;
+
   if (!builder_git_shallow_mirror_ref (location,
                                        flatpak_file_get_path_cached (mirror_dir),
                                        flags,
@@ -481,6 +499,14 @@ builder_source_git_class_init (BuilderSourceGitClass *klass)
                                                          "",
                                                          FALSE,
                                                          G_PARAM_READWRITE));
+
+  g_object_class_install_property (object_class,
+                                   PROP_DISABLE_LFS,
+                                   g_param_spec_boolean ("disable-lfs",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
 }
 
 static void