Merge branch 'master' into gitrefs

Author: Ian2020

.github/ISSUE_TEMPLATE/bug_template.yml

@@ -10,7 +10,7 @@ body:
       description: >-
         What version of flatpak-builder are you using?
         If unsure, run `flatpak-builder --version` in the terminal.
-      placeholder: 1.2.0
+      placeholder: 1.4.1
     validations:
       required: true
 
@@ -43,6 +43,7 @@ body:
         - pip/flatpak-pip-generator
         - poetry/flatpak-poetry-generator.py
         - rubygems/flatpak_rubygems_generator.rb
+        - spm/flatpak-spm-generator.swift
         - yarn/flatpak-yarn-generator.py
     validations:
       required: true

.github/workflows/node.yaml

@@ -18,13 +18,13 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - '3.7'
           - '3.8'
           - '3.9'
           - '3.10'
           - '3.11'
+          - '3.12'
         poetry-version:
-          - '1.1.13'
+          - '1.6.1'
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
@@ -35,7 +35,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - uses: abatilo/actions-poetry@v2.1.6
+      - uses: abatilo/actions-poetry@v2.3.0
         with:
           poetry-version: ${{ matrix.poetry-version }}
       - name: Install OS dependencies

CODEOWNERS

@@ -0,0 +1,2 @@
+/spm/ @david-swift
+/gradle/ @hadess

cargo/README.md

@@ -2,18 +2,21 @@
 
 Tool to automatically generate `flatpak-builder` manifest json from a `Cargo.lock`.
 
-## Requirements:
+## Requirements
 
-Python 3.8+ with following modules:
+Poetry users can run `poetry install` and skip this.
 
+Otherwise install Python 3.8+ with these modules:
 - toml
 - aiohttp
 
 Generated manifests are supported by flatpak-builder 1.2.x or newer.
 
-## Usage:
+## Usage
 
-The first step is to convert the locked dependencies by Cargo into a format flatpak-builder can understand
+Poetry users: first activate your virtualenv by running `poetry shell`.
+
+Convert the locked dependencies by Cargo into a format flatpak-builder can understand:
 ```
 python3 ./flatpak-cargo-generator.py ./quickstart/Cargo.lock -o cargo-sources.json
 ```
@@ -26,7 +29,7 @@ The output file should be added to the manifest like
     "build-commands": [
         "cargo --offline fetch --manifest-path Cargo.toml --verbose",
         "cargo --offline build --release --verbose",
-        "install -Dm755 ./target/debug/quickstart -t /app/bin/"
+        "install -Dm755 ./target/release/quickstart -t /app/bin/"
     ],
     "sources": [
         {
@@ -42,3 +45,4 @@ Make sure to override CARGO_HOME env variable to point it to `/run/build/$module
 
 
 For a complete example see the quickstart project.
+

cargo/flatpak-cargo-generator.py

@@ -114,8 +114,51 @@ def fetch_git_repo(git_url: str, commit: str) -> str:
     if head[:COMMIT_LEN] != commit[:COMMIT_LEN]:
         subprocess.run(['git', 'fetch', 'origin', commit], cwd=clone_dir, check=True)
         subprocess.run(['git', 'checkout', commit], cwd=clone_dir, check=True)
+
+    # Get the submodules as they might contain dependencies. This is a noop if
+    # there are no submodules in the repository
+    subprocess.run(['git', 'submodule', 'update', '--init', '--recursive'], cwd=clone_dir, check=True)
+
     return clone_dir
 
+def update_workspace_keys(pkg, workspace):
+    for key, item in pkg.items():
+        # There cannot be a 'workspace' key if the item is not a dict.
+        if not isinstance(item, dict):
+            continue;
+
+        # Recurse for keys under target.cfg(..)
+        if key == 'target':
+            for target in item.values():
+                update_workspace_keys(target, workspace)
+            continue;
+        # dev-dependencies and build-dependencies should reference root dependencies table from workspace
+        elif key == 'dev-dependencies' or key == 'build-dependencies':
+            update_workspace_keys(item, workspace.get('dependencies', None))
+            continue;
+
+        if not workspace or not key in workspace:
+            continue;
+
+        workspace_item = workspace[key]
+
+        if 'workspace' in item:
+            if isinstance(workspace_item, dict):
+                del item['workspace']
+
+                for dep_key, workspace_value in workspace_item.items():
+                    # features are additive
+                    if dep_key == 'features' and 'features' in item:
+                        item['features'] += workspace_value
+                    else:
+                        item[dep_key] = workspace_value
+            elif len(item) > 1:
+                del item['workspace']
+                item.update({ 'version': workspace_item })
+            else:
+                pkg[key] = workspace_item
+        else:
+            update_workspace_keys(item, workspace_item)
 
 class _GitPackage(NamedTuple):
     path: str
@@ -127,18 +170,10 @@ def normalized(self) -> _TomlType:
         package = copy.deepcopy(self.package)
         if self.workspace is None:
             return package
-        for section_key, section in package.items():
-            # XXX We ignore top-level lists here; maybe we should iterate over list items, too
-            if not isinstance(section, dict):
-                continue
-            for key, value in section.items():
-                if not isinstance(value, dict):
-                    continue
-                if not value.get('workspace'):
-                    continue
-                package[section_key][key] = self.workspace[section_key][key]
-        return package
 
+        update_workspace_keys(package, self.workspace)
+
+        return package
 
 _GitPackagesType = Dict[str, _GitPackage]
 
@@ -148,13 +183,27 @@ async def get_git_repo_packages(git_url: str, commit: str) -> _GitPackagesType:
     git_repo_dir = fetch_git_repo(git_url, commit)
     packages: _GitPackagesType = {}
 
+    def get_cargo_toml_packages(root_dir: str, workspace: Optional[_TomlType] = None):
+        assert not os.path.isabs(root_dir) and os.path.isdir(root_dir)
+
+        with workdir(root_dir):
+            if os.path.exists('Cargo.toml'):
+                cargo_toml = load_toml('Cargo.toml')
+                workspace = cargo_toml.get('workspace') or workspace
+
+                if 'package' in cargo_toml:
+                    packages[cargo_toml['package']['name']] = _GitPackage(
+                        path=os.path.normpath(root_dir),
+                        package=cargo_toml,
+                        workspace=workspace
+                    )
+        for child in os.scandir(root_dir):
+            if child.is_dir():
+                # the workspace can be referenced by any subdirectory
+                get_cargo_toml_packages(child.path, workspace)
+
     with workdir(git_repo_dir):
-        if os.path.isfile('Cargo.toml'):
-            packages.update(await get_cargo_toml_packages(load_toml('Cargo.toml'), '.'))
-        else:
-            for toml_path in glob.glob('*/Cargo.toml'):
-                packages.update(await get_cargo_toml_packages(load_toml(toml_path),
-                                                              os.path.dirname(toml_path)))
+        get_cargo_toml_packages('.')
 
     assert packages, f"No packages found in {git_repo_dir}"
     logging.debug(
@@ -168,68 +217,6 @@ async def get_git_repo_packages(git_url: str, commit: str) -> _GitPackagesType:
     return packages
 
 
-async def get_cargo_toml_packages(root_toml: _TomlType, root_dir: str) -> _GitPackagesType:
-    assert not os.path.isabs(root_dir) and os.path.isdir(root_dir)
-    assert 'package' in root_toml or 'workspace' in root_toml
-    packages: _GitPackagesType = {}
-
-    async def get_dep_packages(
-        entry: _TomlType,
-        toml_dir: str,
-        workspace: Optional[_TomlType] = None,
-    ):
-        assert not os.path.isabs(toml_dir)
-        # https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html
-        if 'dependencies' in entry:
-            for dep_name, dep in entry['dependencies'].items():
-                if 'package' in dep:
-                    dep_name = dep['package']
-                if 'path' not in dep:
-                    continue
-                if dep_name in packages:
-                    continue
-                dep_dir = os.path.normpath(os.path.join(toml_dir, dep['path']))
-                logging.debug("Loading dependency %s from %s", dep_name, dep_dir)
-                dep_toml = load_toml(os.path.join(dep_dir, 'Cargo.toml'))
-                assert dep_toml['package']['name'] == dep_name, toml_dir
-                await get_dep_packages(dep_toml, dep_dir, workspace)
-                packages[dep_name] = _GitPackage(
-                    path=dep_dir,
-                    package=dep,
-                    workspace=workspace,
-                )
-        if 'target' in entry:
-            for _, target in entry['target'].items():
-                await get_dep_packages(target, toml_dir)
-
-    if 'package' in root_toml:
-        await get_dep_packages(root_toml, root_dir)
-        packages[root_toml['package']['name']] = _GitPackage(
-            path=root_dir,
-            package=root_toml,
-            workspace=None,
-        )
-
-    if 'workspace' in root_toml:
-        for member in root_toml['workspace'].get('members', []):
-            for subpkg_toml in glob.glob(os.path.join(root_dir, member, 'Cargo.toml')):
-                subpkg = os.path.normpath(os.path.dirname(subpkg_toml))
-                logging.debug(
-                    "Loading workspace member %s in %s",
-                    subpkg_toml,
-                    os.path.abspath(root_dir),
-                )
-                pkg_toml = load_toml(subpkg_toml)
-                await get_dep_packages(pkg_toml, subpkg, root_toml['workspace'])
-                packages[pkg_toml['package']['name']] = _GitPackage(
-                    path=subpkg,
-                    package=pkg_toml,
-                    workspace=root_toml['workspace'],
-                )
-
-    return packages
-
-
 _FlatpakSourceType = Dict[str, Any]
 
 

cargo/pyproject.toml

@@ -0,0 +1,12 @@
+[tool.poetry]
+package-mode = false
+
+[tool.poetry.dependencies]
+python = ">=3.8"
+aiohttp = "^3.9.5"
+toml = "^0.10.2"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

cargo/quickstart/com.flatpak.quickstart.json

@@ -25,7 +25,7 @@
         "build-commands": [
             "cargo --offline fetch --manifest-path Cargo.toml --verbose",
             "cargo --offline build --release --verbose",
-            "install -Dm755 ./target/debug/qucikstart -t /app/bin/"
+            "install -Dm755 ./target/release/quickstart -t /app/bin/"
         ],
         "sources": [{
                 "type": "dir",

dotnet/README.md

@@ -32,4 +32,17 @@ If you want to change the directory name, run `flatpak-dotnet-generator.py` with
 - `--runtime` or `-r` The target [runtime](https://learn.microsoft.com/en-us/dotnet/core/rid-catalog#linux-rids) to restore packages for.
 - `--dotnet` or `-d` The target version of dotnet to use. (Defaults to latest LTS version)
 - `--freedesktop` or `-f` The target version of the freedesktop sdk to use. (Defaults to latest version)
-- `--destdir` The directory the generated sources file will save sources to `nuget-sources` by default. 
+- `--destdir` The directory the generated sources file will save sources to `nuget-sources` by default.
+- `--dotnet-args` or `-a` Pass additional arguments to the `dotnet` command.
+
+## Example
+
+To pass multiple arguments to the `dotnet` command, use the `--dotnet-args` option:
+
+```bash
+python3 flatpak-dotnet-generator.py my-output-sources.json my.input.Desktop.csproj --runtime linux-x64 --dotnet-args --no-cache --verbosity detailed
+```
+
+In this example:
+- `--no-cache` and `--verbosity detailed` are additional arguments passed to the `dotnet` command.
+- You can add as many arguments as needed after `--dotnet-args`.

dotnet/flatpak-dotnet-generator.py

@@ -10,42 +10,50 @@
 import json
 import subprocess
 import tempfile
-
+import concurrent.futures
 
 def main():
     # Bump this to latest freedesktop runtime version.
-    freedesktop_default = '22.08'
+    freedesktop_default = '24.08'
     # Bump this to an LTS dotnet version.
-    dotnet_default = '6'
+    dotnet_default = '8'
 
     parser = argparse.ArgumentParser()
     parser.add_argument('output', help='The output JSON sources file')
-    parser.add_argument('project', help='The project file')
-    parser.add_argument('--runtime', '-r', help='The target runtime to restore packages for')
+    parser.add_argument('project', nargs='+', help='The project file(s)')
+    parser.add_argument('--runtime', '-r', nargs='+', default=[None], help='The target runtime(s) to restore packages for')
     parser.add_argument('--freedesktop', '-f', help='The target version of the freedesktop sdk to use', 
                         default=freedesktop_default)
     parser.add_argument('--dotnet', '-d', help='The target version of dotnet to use', 
                         default=dotnet_default)
     parser.add_argument('--destdir',
                         help='The directory the generated sources file will save sources to',
                         default='nuget-sources')
+    parser.add_argument('--dotnet-args', '-a', nargs=argparse.REMAINDER, 
+                        help='Additional arguments to pass to the dotnet command')
     args = parser.parse_args()
 
     sources = []
-
     with tempfile.TemporaryDirectory(dir=Path()) as tmp:
-        runtime_args = []
-        if args.runtime:
-            runtime_args.extend(('-r', args.runtime))
+        def restore_project(project, runtime):
+            subprocess.run([
+                'flatpak', 'run',
+                '--env=DOTNET_CLI_TELEMETRY_OPTOUT=true',
+                '--env=DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true',
+                '--command=sh', f'--runtime=org.freedesktop.Sdk//{args.freedesktop}', '--share=network',
+                '--filesystem=host', f'org.freedesktop.Sdk.Extension.dotnet{args.dotnet}//{args.freedesktop}', '-c',
+                f'PATH="${{PATH}}:/usr/lib/sdk/dotnet{args.dotnet}/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/sdk/dotnet{args.dotnet}/lib" exec dotnet restore "$@"',
+                '--', '--packages', tmp, project] + (['-r', runtime] if runtime else []) + (args.dotnet_args or []))
 
-        subprocess.run([
-            'flatpak', 'run',
-            '--env=DOTNET_CLI_TELEMETRY_OPTOUT=true',
-            '--env=DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true',
-            '--command=sh', f'--runtime=org.freedesktop.Sdk//{args.freedesktop}', '--share=network',
-            '--filesystem=host', f'org.freedesktop.Sdk.Extension.dotnet{args.dotnet}//{args.freedesktop}', '-c',
-            f'PATH="${{PATH}}:/usr/lib/sdk/dotnet{args.dotnet}/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/sdk/dotnet{args.dotnet}/lib" exec dotnet restore "$@"',
-            '--', '--packages', tmp, args.project] + runtime_args)
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = []
+            for project in args.project:
+                if args.runtime:
+                    for runtime in args.runtime:
+                        futures.append(executor.submit(restore_project, project, runtime))
+                else:
+                    futures.append(executor.submit(restore_project, project, None))
+            concurrent.futures.wait(futures)
 
         for path in Path(tmp).glob('**/*.nupkg.sha512'):
             name = path.parent.parent.name

go-get/README.md

@@ -1,5 +1,8 @@
 # Flatpak Go Get Generator
-Tool to automatically create the source list for a Go module.
+Tool to automatically create the source list for a Go module (legacy).
+
+It runs the build in a legacy `GOPATH` mode.
+For a module-aware mode, see [go-modules](../go-modules/README.md) or [flatpak-go-vendor-generator](./flatpak-go-vendor-generator.py) script.
 
 The script does not require Go in the host system.