Fix patches missing in Flatpak nightly

Author: theCapypara

installer/linux-flatpak/.gitignore

@@ -3,4 +3,5 @@ build-dir/
 cargo-sources*.json
 requirements-*.json
 org.skytemple.*.yml
-stable/
+stable/
+tmp-downloads/

installer/linux-flatpak/generator--download-repo-with-submodules.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Creates a tar.gz from a Git repo.
+
+# Based on: https://gist.github.com/bojanpotocnik/e18c90f5993dca7ea495ce82bba1364f
+
+DESTINATION=$2
+cd $1
+REPO_DIR="$(git rev-parse --show-toplevel)"
+TOP_PREFIX=$(basename "$REPO_DIR")
+TOP_FILE="$REPO_DIR/$TOP_PREFIX-$(git rev-parse --short HEAD).tar"
+
+# Update all submodules
+git submodule update --init --recursive
+
+# Create module archive
+git archive --prefix="$TOP_PREFIX/" --output="$TOP_FILE" HEAD
+
+# Archive each submodule recursively:
+# 1. Create .sub.tar file for each submodule using correct relative paths
+# 2. Concatenate each .sub.tar file into a top-most .tar file
+# 3. Remove each .sub.tar file afterwards
+# Note:
+# - $displaypath contains the relative path from the current working directory to the submodules root directory
+# - $toplevel is the absolute path to the top-level of the immediate super-project
+# - Use single instead of double quotes to expand $displaypath within the git submodule foreach command, not here
+# - Export other variables so that they can be expanded within the git submodule foreach command
+export TOP_PREFIX
+export TOP_FILE
+# shellcheck disable=SC2016
+git submodule foreach --recursive '
+    SUB_PREFIX="$TOP_PREFIX/$displaypath/" &&
+    SUB_FILE="$(basename $(git rev-parse --show-toplevel))-$(git rev-parse --short HEAD).tar" &&
+    git archive --prefix="$SUB_PREFIX" --output="$SUB_FILE" HEAD &&
+    tar --concatenate --file="$TOP_FILE" "$SUB_FILE" &&
+    echo "    Merged $SUB_FILE to $TOP_FILE" &&
+    rm "$SUB_FILE"
+'
+
+# Compress the combined tar file
+printf "\nCompressing...\n"
+gzip --best --verbose "$TOP_FILE"
+
+mv "${TOP_FILE}.gz" "$DESTINATION"

installer/linux-flatpak/generator.py

@@ -20,7 +20,6 @@
 from shutil import which
 from tempfile import TemporaryDirectory
 from typing import Iterator, Literal, Required, TypedDict, cast
-from urllib.request import urlretrieve
 
 import requirements
 import yaml
@@ -65,6 +64,7 @@ class FlatpakSourceRef(TypedDict, total=False):
 
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+CACHED_DOWNLOADS_DIR = "tmp-downloads"
 INTERPRETER = sys.executable
 REQUIREMENTS_FILE_NIGHTLY = os.path.join(BASE_DIR, "..", "..", "requirements.txt")
 REQUIREMENTS_FILE_STABLE = os.path.join(BASE_DIR, "..", "..", "requirements-frozen.txt")
@@ -205,8 +205,11 @@ def source_ref_skytemple_randomizer(git_tag: str | None) -> FlatpakSourceRef:
 
 
 @contextmanager
-def clone_repo_for_generator(req: Requirement, git_url: str) -> Iterator[str]:
+def clone_repo_for_generator(req: Requirement, git_url: str, dirname: str | None = None) -> Iterator[str]:
     with TemporaryDirectory() as tmp_dir:
+        if dirname is not None:
+            tmp_dir = os.path.join(tmp_dir, dirname)
+            os.makedirs(tmp_dir)
         print_t(f"Cloning repo for {req.name} for analysis to tmp dir {tmp_dir}")
         tag = get_git_tag_for_req(req)
         run_git("clone", git_url, tmp_dir)
@@ -222,7 +225,7 @@ def get_pkgname_and_version_from_repo(req: Requirement, git_url: str) -> tuple[s
         return get_pkgname_and_version_from_pyproject_toml(os.path.join(tmp_dir, "pyproject.toml"))
 
 
-def fixup_and_add_extra_pip_dependencies(out_json_path: str, extras: dict) -> None:
+def fixup_and_add_extra_pip_dependencies(out_json_path: str, extras: dict, cache_dir: str) -> None:
     print_t("Fixing up requirements JSON file and adding additional dependencies")
     with open(out_json_path, "r") as f:
         modules = json.load(f)
@@ -254,37 +257,49 @@ def remove_module(name: str, mods: list[str]):
         ]
         # The pip generator doesn't properly parse the @ part of VCS dependencies, always trying to attribute
         # it as a commit. Sigh.
-        # Also installing Git repos only works for one single depedency, so use download tar.gz instead.
+        # Also installing Git repos only works for one single dependency, so we build a tar.gz with submodules in it.
         for source in module["sources"]:
             if "commit" in source:
                 if source["commit"] is None:
                     branch = "master"
                 else:
                     branch = source["commit"]
                 giturl = source["url"]
+                pseudo_req = Requirement.parse_line(f"{giturl} @ {giturl}@{branch}")
                 if giturl not in version_cache or giturl not in names_cache:
-                    names_cache[giturl], version_cache[giturl] = get_pkgname_and_version_from_repo(
-                        Requirement.parse_line(f"{giturl} @ {giturl}@{branch}"), giturl
-                    )
+                    names_cache[giturl], version_cache[giturl] = get_pkgname_and_version_from_repo(pseudo_req, giturl)
                 # Make sure to install the requirement from the downloaded tar instead
                 if f"python3-{names_cache[giturl]}" == module["name"]:
                     module["build-commands"][0] = (
                         module["build-commands"][0].removesuffix('"."') + f'"{names_cache[giturl]}"'
                     )
                 source["type"] = "file"
                 del source["commit"]
-                source["url"] = source["url"] + f"/archive/refs/heads/{branch}.tar.gz"
+                fname = f"{names_cache[giturl]}-{version_cache[giturl]}.tar.gz"
+                source["url"] = os.path.abspath(os.path.join(cache_dir, fname))
                 if source["url"] not in checksum_cache:
-                    # download to get checksum
-                    with TemporaryDirectory() as tmp_dir:
-                        print_t(f"Fetching {source["url"]} for hashsum.")
-                        fpath = os.path.join(tmp_dir, "file")
-                        urlretrieve(source["url"], fpath)
-                        with open(fpath, "rb", buffering=0) as f9:
+                    # download to generate tarball and to get checksum
+                    with clone_repo_for_generator(
+                        pseudo_req, giturl, f"{names_cache[giturl]}-{version_cache[giturl]}"
+                    ) as tmp_dir:
+                        print_t(f"Fetching {source["url"]} for tar.gz.")
+                        print_cmd(f"generator--download-repo-with-submodules.sh {tmp_dir} {source["url"]}")
+                        subprocess.run(
+                            [
+                                os.path.join(BASE_DIR, "generator--download-repo-with-submodules.sh"),
+                                tmp_dir,
+                                source["url"],
+                            ],
+                            check=True,
+                            capture_output=True,
+                        )
+                        with open(source["url"], "rb", buffering=0) as f9:
                             checksum_cache[source["url"]] = hashlib.file_digest(f9, "sha256").hexdigest()  # type: ignore
 
                 source["sha256"] = checksum_cache[source["url"]]
-                source["dest-filename"] = f"{names_cache[giturl]}-{version_cache[giturl]}.tar.gz"
+                source["dest-filename"] = fname
+                source["path"] = source["url"]
+                del source["url"]
     with open(out_json_path, "w") as f3:
         json.dump(modules, f3, indent=4)
 
@@ -304,6 +319,7 @@ def pip_add_group(extras: dict, value: str, dest: str | None = None) -> str:
 def generate(out_dir: str, reqs_file: str, *, tag: str | None = None) -> None:
     reqs = parse_reqs_with_skytemple_files_reqs(reqs_file)
     print_t(f"Generating to {out_dir}. Tag info: {tag}")
+    os.makedirs(os.path.join(out_dir, CACHED_DOWNLOADS_DIR), exist_ok=True)
     with open(os.path.join(BASE_DIR, MANIFEST_TEMPLATE_FILE), "r") as f:
         jinja_env = Environment(loader=DictLoader({"tmpl": f.read()}))
 
@@ -331,7 +347,11 @@ def generate(out_dir: str, reqs_file: str, *, tag: str | None = None) -> None:
                 os.path.join(path_skytemple_rust, "Cargo.lock"),
                 os.path.join(out_dir, "cargo-sources-skytemple-rust.json"),
             )
-            fixup_and_add_extra_pip_dependencies(os.path.join(out_dir, "requirements-skytemple-randomizer.json"), extras)
+            fixup_and_add_extra_pip_dependencies(
+                os.path.join(out_dir, "requirements-skytemple-randomizer.json"),
+                extras,
+                os.path.join(out_dir, CACHED_DOWNLOADS_DIR),
+            )
 
     print_t("Writing manifest file...")
     with open(os.path.join(out_dir, OUT_MANIFEST_FILE), "w") as f: