builder-git: Add support for fetching and checking out Git LFS objects

This explicitly fetches Git LFS objects as long as git-lfs is available
on host (but doesn't need to be configured)

Fixes #463

Co-authored-by: Bartłomiej Piotrowski <b@bpiotrowski.pl>

Author: bbhtt

.github/workflows/check.yml

@@ -26,7 +26,7 @@ jobs:
         libseccomp-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \
         libgirepository1.0-dev libappstream-dev libdconf-dev clang socat flatpak \
         libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch libarchive-tools \
-        docbook-xsl xmlto xsltproc
+        docbook-xsl xmlto xsltproc git-lfs
     - name: Check out flatpak
       uses: actions/checkout@v4
       with:
@@ -59,7 +59,7 @@ jobs:
         libseccomp-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \
         libgirepository1.0-dev libappstream-dev libdconf-dev clang flatpak \
         libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch libarchive-tools \
-        docbook-xsl xmlto xsltproc
+        docbook-xsl xmlto xsltproc git-lfs
     - name: Check out flatpak
       uses: actions/checkout@v4
       with:
@@ -135,7 +135,8 @@ jobs:
           libarchive-tools \
           docbook-xsl \
           xmlto \
-          xsltproc
+          xsltproc \
+          git-lfs
 
     - name: Check out flatpak-builder
       uses: actions/checkout@v4

README.md

@@ -41,6 +41,7 @@ Very commonly used:
  * git
  * 7z
  * bsdunzip (libarchive)
+ * git-lfs
 
 Rarely used:
 

src/builder-git.c

@@ -148,6 +148,85 @@ git_has_version (int major,
   return TRUE;
 }
 
+static gboolean
+git_lfs (GFile   *dir,
+         char   **output,
+         GSubprocessFlags flags,
+         GError **error,
+         const char **args)
+{
+  g_autoptr(GPtrArray) full_args = g_ptr_array_new ();
+  gboolean res;
+
+  g_ptr_array_add (full_args, "git");
+  g_ptr_array_add (full_args, "lfs");
+
+  for (const char **arg = args; *arg; arg++)
+    g_ptr_array_add (full_args, (char *) *arg);
+
+  g_ptr_array_add (full_args, NULL);
+
+  res = flatpak_spawnv (dir, output, flags, error, (const char **) full_args->pdata, NULL);
+
+  return res;
+}
+
+static gboolean
+git_lfs_is_available (void)
+{
+  static gsize lfs_available = 0;
+
+  if (g_once_init_enter (&lfs_available))
+    {
+      g_autoptr(GError) my_error = NULL;
+      gsize new_lfs_available = 2;
+
+      if (git_lfs (NULL, NULL, 0, &my_error, (const char *[]){"version", NULL}))
+        new_lfs_available = 1;
+      else
+        g_debug ("git-lfs is not available: %s", my_error->message);
+
+      g_once_init_leave (&lfs_available, new_lfs_available);
+    }
+
+  return lfs_available == 1;
+}
+
+static gboolean
+builder_git_run_lfs (GFile       *dir,
+                     GError     **error,
+                     const char  *subcommand,
+                     ...)
+{
+  va_list ap;
+  g_autoptr(GPtrArray) args = g_ptr_array_new ();
+  const char *arg;
+  gboolean ok;
+
+  if (!git_lfs_is_available ())
+    return TRUE;
+
+  g_ptr_array_add (args, (char *) subcommand);
+
+  va_start (ap, subcommand);
+  while ((arg = va_arg (ap, const char *)))
+    g_ptr_array_add (args, (char *) arg);
+  va_end (ap);
+
+  g_ptr_array_add (args, NULL);
+
+  g_print ("Running git lfs %s\n", subcommand);
+  ok = git_lfs (dir, NULL, 0, error, (const char **) args->pdata);
+
+  if (!ok)
+    {
+      git_lfs (dir, NULL, 0, NULL, (const char *[]){"logs", "last", NULL});
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
 static gboolean
 git_version_supports_fsck_and_shallow (void)
 {
@@ -578,6 +657,10 @@ builder_git_mirror_repo (const char     *repo_location,
                     origin, full_ref_mapping, NULL))
             return FALSE;
 
+          if (!builder_git_run_lfs (mirror_dir, error,
+                                    "fetch", "--all", NULL))
+            return FALSE;
+
 	  /* It turns out that older versions of git (at least 2.7.4)
 	   * cannot check out a commit unless a real tag/branch points
 	   * to it, which is not the case for e.g. gitbug pull requests.
@@ -606,6 +689,10 @@ builder_git_mirror_repo (const char     *repo_location,
                     was_shallow ? "--unshallow" : NULL,
                     NULL))
             return FALSE;
+
+          if (!builder_git_run_lfs (mirror_dir, error,
+                                    "fetch", "--all", NULL))
+            return FALSE;
         }
 
       if (alternates)
@@ -708,6 +795,10 @@ builder_git_shallow_mirror_ref (const char     *repo_location,
             "fetch", "--depth", "1", "origin", full_ref_colon_full_ref, NULL))
     return FALSE;
 
+  if (!builder_git_run_lfs (mirror_dir, error,
+                            "fetch", "origin", full_ref, NULL))
+    return FALSE;
+
   /* Always mirror submodules */
   current_commit = git_get_current_commit (mirror_dir, ref, FALSE, context, error);
   if (current_commit == NULL)
@@ -862,6 +953,9 @@ builder_git_checkout (const char     *repo_location,
             "checkout", branch, NULL))
     return FALSE;
 
+  if (!builder_git_run_lfs (dest, error, "checkout", NULL))
+    return FALSE;
+
   if (mirror_flags & FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES)
     if (!git_extract_submodule (repo_location, dest, branch, context, error))
       return FALSE;
@@ -897,4 +991,4 @@ builder_git_get_default_branch (const char *repo_location)
 
   g_debug ("Failed to auto-detect default branch from git output");
   return g_strdup ("master");
-}
+}