[mac-ai] Work on the releasing process (#785)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added new environment variables for logging and provider selection in
Podman machine startup scripts.
* Expanded support for multiple GGML backend libraries in remoting
environments and packaging.

* **Bug Fixes**
* Corrected handling of version and branch references in configuration
and version link formatting.
  * Fixed use of mutable default arguments in remote access utilities.

* **Refactor**
* Simplified and unified handling of backend libraries in packaging and
preparation scripts.
* Modularized and improved error handling in update scripts for krunkit
and related binaries.
* Removed legacy `git_ref` logic in favor of streamlined version-based
workflows.

* **Style**
* Standardized informational and error messages in scripts for improved
clarity.

* **Chores**
* Updated test platform configurations and build flags for improved
compatibility and maintainability.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: kpouget

projects/mac_ai/testing/config.yaml

@@ -63,8 +63,7 @@ ci_presets:
 
   cpt_ramalama:
     extends: [use_intlab_os, analyze_kpis, export_kpis]
-    prepare.ramalama.repo.version: null
-    prepare.ramalama.repo.git_ref: main
+    prepare.ramalama.repo.version: main
     prepare.ramalama.build_image.enabled: true
 
     test.platform: [podman/ramalama]
@@ -95,22 +94,20 @@ ci_presets:
   remoting_publish:
     extends: [remoting]
     prepare.remoting.publish: true
-    prepare.llama_cpp.source.repo.version: b5709-v250627
-    prepare.virglrenderer.repo.branch: v25-06-18
-    prepare.ramalama.build_image.enabled: "remoting"
+    prepare.llama_cpp.source.repo.version: b5709-v250701.0
+    prepare.virglrenderer.repo.branch: v1.1.1-remoting-0.1
+    prepare.ramalama.build_image.enabled: true
     prepare.ramalama.build_image.publish.enabled: true
     prepare.ramalama.build_image.registry_path: quay.io/crcont
     prepare.ramalama.build_image.name: remoting
     prepare.ramalama.repo.url: https://github.com/kpouget/ramalama
-    prepare.ramalama.repo.git_ref: v0.9.3-remoting-rc5
-    prepare.ramalama.repo.version: null
+    prepare.ramalama.repo.version: v0.10.0-remoting-0.2.0
 
     test.platform:
     - podman/ramalama/remoting
     - podman/llama_cpp/remoting
     - macos/llama_cpp/metal
-    - macos/llama_cpp/vulkan
-    # - podman/llama_cpp/vulkan # not working today
+    - podman/llama_cpp/vulkan
 
   remoting_testing:
     extends: [remoting]
@@ -310,11 +307,11 @@ prepare:
     repo:
       url: https://github.com/containers/ramalama
       version: latest
-      git_ref:
     build_image:
       enabled: false
       name: ramalama
       registry_path: localhost
+      debug: false
       publish:
         enabled: false
         credentials: "*[email protected]_registry"
@@ -345,7 +342,7 @@ prepare:
       url: https://gitlab.freedesktop.org/kpouget/virglrenderer
       branch: main
     build:
-      flags: -Dvenus=true -Dc_link_args=-L/opt/homebrew/lib/
+      flags: -Dapir=true -Dc_link_args=-L/opt/homebrew/lib/
 
   podman:
     repo:
@@ -385,10 +382,10 @@ prepare:
         CONTAINERS_HELPER_BINARY_DIR: /opt/homebrew/bin/
       remoting_env:
         enabled: false
-        apir_lib:
-          name: libggml-remotingbackend.dylib
-        ggml_lib:
-          name: libggml-metal.dylib
+        ggml_libs:
+        - libggml-remotingbackend.dylib # keep the APIR backend first
+        - libggml-metal.dylib # keep the GGML backend second
+        - libggml-base.dylib
         env:
           APIR_LLAMA_CPP_GGML_LIBRARY_REG: ggml_backend_metal_reg
           APIR_LLAMA_CPP_GGML_LIBRARY_INIT: ggml_backend_metal_init

projects/mac_ai/testing/podman_machine.py

@@ -47,14 +47,14 @@ def start(base_work_dir, use_remoting=None):
     if use_remoting:
         env["DYLD_LIBRARY_PATH"] = prepare_virglrenderer.get_dyld_library_path(base_work_dir) # not working ... (blocked by MacOS when SSHing ...)
         llama_remoting_backend_build_dir = prepare_llama_cpp.get_remoting_build_dir(base_work_dir)
-        env["VIRGL_APIR_BACKEND_LIBRARY"] = llama_remoting_backend_build_dir / config.project.get_config("prepare.podman.machine.remoting_env.apir_lib.name")
-        env["APIR_LLAMA_CPP_GGML_LIBRARY_PATH"] = llama_remoting_backend_build_dir / config.project.get_config("prepare.podman.machine.remoting_env.ggml_lib.name")
+        env["VIRGL_APIR_BACKEND_LIBRARY"] = llama_remoting_backend_build_dir / config.project.get_config("prepare.podman.machine.remoting_env.ggml_libs[0]")
+        env["APIR_LLAMA_CPP_GGML_LIBRARY_PATH"] = llama_remoting_backend_build_dir / config.project.get_config("prepare.podman.machine.remoting_env.ggml_libs[1]")
         env |= config.project.get_config("prepare.podman.machine.remoting_env.env")
         prepare_virglrenderer.configure(base_work_dir, use_custom=True)
     else:
         prepare_virglrenderer.configure(base_work_dir, use_custom=False)
 
-    ret = _run(base_work_dir, f"start {name}", env, print_cmd=True)
+    ret = _run(base_work_dir, f"start {name} --no-info", env, print_cmd=True)
 
     if use_remoting and config.project.get_config("prepare.podman.machine.remoting_env.enabled"):
         if not config.project.get_config("prepare.virglrenderer.enabled"):

projects/mac_ai/testing/prepare_release.py

@@ -10,26 +10,24 @@
 import llama_cpp, ollama, ramalama
 
 def create_remoting_tarball(base_work_dir):
+    package_libs = []
     virglrenderer_lib = prepare_virglrenderer.get_dyld_library_path(base_work_dir, with_lib=True)
 
     if not remote_access.exists(virglrenderer_lib):
         raise ValueError(f"Cannot publish the remoting libraries, {virglrenderer_lib} does not exist")
+    package_libs.append(virglrenderer_lib)
 
     llama_remoting_backend_build_dir = prepare_llama_cpp.get_remoting_build_dir(base_work_dir)
-    apir_backend_lib = llama_remoting_backend_build_dir / config.project.get_config("prepare.podman.machine.remoting_env.apir_lib.name")
-    if not remote_access.exists(apir_backend_lib):
-        raise ValueError(f"Cannot publish the remoting libraries, {apir_backend_lib} does not exist")
 
-    llama_cpp_backend_lib = llama_remoting_backend_build_dir / config.project.get_config("prepare.podman.machine.remoting_env.ggml_lib.name")
-    if not remote_access.exists(llama_cpp_backend_lib):
-        raise ValueError(f"Cannot publish the remoting libraries, {llama_cpp_backend_lib} does not exist")
-
-    virglrenderer_branch = config.project.get_config("prepare.virglrenderer.repo.branch")
-    if not virglrenderer_branch.startswith("v"):
-        raise ValueError("Cannot publish the remoting libraries, virglrenderer not built from a released version")
+    ggml_backend_libs = config.project.get_config("prepare.podman.machine.remoting_env.ggml_libs")
+    for libname in ggml_backend_libs:
+        backend_lib = llama_remoting_backend_build_dir / libname
+        if not remote_access.exists(backend_lib):
+            raise ValueError(f"Cannot publish the remoting libraries, {backend_lib} does not exist")
+        package_libs.append(backend_lib)
 
     with env.NextArtifactDir("build_remoting_tarball"):
-        return build_remoting_tarball(base_work_dir, virglrenderer_lib, llama_cpp_backend_lib, apir_backend_lib)
+        return build_remoting_tarball(base_work_dir, package_libs)
 
 
 def add_string_file(dest, content):
@@ -68,7 +66,7 @@ def add_remote_git_status(base_work_dir, src_dir, dest):
 
 
 def get_version_link(repo_url, version, git_rev, github=False, gitlab=False):
-    url = f"{repo_url}/-/commit/{git_rev} ({version})" if gitlab \
+    url = f"{repo_url}/-/commit/{git_rev}" if gitlab \
         else f"{repo_url}/commit/{git_rev}"
 
     if version:
@@ -77,11 +75,11 @@ def get_version_link(repo_url, version, git_rev, github=False, gitlab=False):
     return url
 
 
-def build_remoting_tarball(base_work_dir, virglrenderer_lib, llama_cpp_backend_lib, apir_backend_lib):
+def build_remoting_tarball(base_work_dir, package_libs):
     llama_cpp_version = config.project.get_config("prepare.llama_cpp.source.repo.version")
     llama_cpp_url = config.project.get_config("prepare.llama_cpp.source.repo.url")
 
-    ramalama_version = config.project.get_config("prepare.ramalama.repo.git_ref")
+    ramalama_version = config.project.get_config("prepare.ramalama.repo.version")
 
     virglrenderer_version = config.project.get_config("prepare.virglrenderer.repo.branch")
     virglrenderer_url = config.project.get_config("prepare.virglrenderer.repo.url")
@@ -96,8 +94,6 @@ def build_remoting_tarball(base_work_dir, virglrenderer_lib, llama_cpp_backend_l
     src_info_dir = tarball_dir / "src_info"
     src_info_dir.mkdir()
 
-    add_remote_file(base_work_dir, virglrenderer_lib, bin_dir / virglrenderer_lib.name)
-
     virglrenderer_src_dir = prepare_virglrenderer.get_build_dir(base_work_dir) / ".." / "src"
     virglrenderer_git_rev = add_remote_git_status(base_work_dir, virglrenderer_src_dir,
                                                   src_info_dir / "virglrenderer.git-commit.txt")
@@ -110,9 +106,8 @@ def build_remoting_tarball(base_work_dir, virglrenderer_lib, llama_cpp_backend_l
 
     llama_cpp_version_link = get_version_link(llama_cpp_url, llama_cpp_version, llama_cpp_git_rev, github=True)
 
-    add_remote_file(base_work_dir, llama_cpp_backend_lib, bin_dir / llama_cpp_backend_lib.name)
-
-    add_remote_file(base_work_dir, apir_backend_lib, bin_dir / apir_backend_lib.name)
+    for backend_lib in package_libs:
+        add_remote_file(base_work_dir, backend_lib, bin_dir / backend_lib.name)
 
     machine_script_file = pathlib.Path("projects/mac_ai/testing/scripts/podman_start_machine.api_remoting.sh")
     add_local_file(machine_script_file, tarball_dir / machine_script_file.name)

projects/mac_ai/testing/prepare_virglrenderer.py

@@ -7,6 +7,7 @@
 
 def get_build_dir(base_work_dir):
     version = config.project.get_config("prepare.virglrenderer.repo.branch")
+
     return base_work_dir / "virglrenderer" / version / "build"
 
 
@@ -28,6 +29,12 @@ def prepare(base_work_dir):
     repo_url = config.project.get_config("prepare.virglrenderer.repo.url")
     build_flags = config.project.get_config("prepare.virglrenderer.build.flags")
     version = config.project.get_config("prepare.virglrenderer.repo.branch")
+    refspec = None
+    if version.startswith("pr-"):
+        pr_number = version.removeprefix("pr-")
+        refspec = f"refs/merge-requests/{pr_number}/head"
+        version = None
+
     build_dir = get_build_dir(base_work_dir)
     src_dir = build_dir.parent / "src"
 
@@ -36,6 +43,7 @@ def prepare(base_work_dir):
         repo_url=repo_url,
         dest=src_dir,
         version=version,
+        refspec=refspec,
         artifact_dir_suffix="__virglrenderer",
         force=True,
     )

projects/mac_ai/testing/ramalama.py

@@ -17,30 +17,23 @@ def prepare_test(base_work_dir, platform):
 
 
 def _get_binary_path(base_work_dir, platform):
-    git_ref = config.project.get_config("prepare.ramalama.repo.git_ref")
     version = config.project.get_config("prepare.ramalama.repo.version")
 
-    if git_ref and version:
-        raise ValueError(f"Cannot have Ramalama git_ref={git_ref} and version={version} set together.")
-
     if version == "latest":
         repo_url = config.project.get_config("prepare.ramalama.repo.url")
         version = utils.get_latest_release(repo_url)
         config.project.set_config("prepare.ramalama.repo.version", version)
 
-    system_file = f"{version}.zip"
-
     dest_base = base_work_dir / "ramalama-ai"
-    if version:
-        # don't use 'ramalama' in the base_work_dir, otherwise Python
-        # takes it (invalidly) for the package `ramalama` package
-        dest = dest_base / system_file
-        dest_dir = dest.parent / f"ramalama-{version.removeprefix('v')}"
-    else:
-        dest = dest_dir = dest_base / f"ramalama-{git_ref}"
 
-    ramalama_path =  dest_dir / "bin" / "ramalama"
-    return ramalama_path, dest, (version, git_ref)
+    # don't use 'ramalama' in the base_work_dir, otherwise Python
+    # takes it (invalidly) for the package `ramalama` package
+
+    dest_dir = dest_base / f"ramalama-{version.removeprefix('v')}"
+
+    ramalama_path = dest_dir / "bin" / "ramalama"
+
+    return ramalama_path, dest_dir, version
 
 
 def get_binary_path(base_work_dir, platform):
@@ -52,40 +45,28 @@ def get_binary_path(base_work_dir, platform):
 
     return ramalama_path
 
-def download_ramalama(base_work_dir, dest, version, git_ref):
+def download_ramalama(base_work_dir, dest, version):
     repo_url = config.project.get_config("prepare.ramalama.repo.url")
 
-    if version:
-        source = "/".join([
-            repo_url,
-            "archive/refs/tags",
-            f"{version}.zip",
-        ])
-        run.run_toolbox(
-            "remote", "download",
-            source=source, dest=dest,
-            tarball=True,
-        )
+    kwargs = dict(
+        repo_url=repo_url,
+        dest=dest,
+    )
+    if version.startswith("pr-"):
+        pr_number = version.removeprefix("pr-")
+        kwargs["refspec"] = f"refs/pull/{pr_number}/head"
     else:
-        kwargs = dict(
-            repo_url=repo_url,
-            dest=dest,
-        )
-        if git_ref.startswith("pr-"):
-            pr_number = git_ref.removeprefix("pr-")
-            kwargs["refspec"] = f"refs/pull/{pr_number}/head"
-        else:
-            kwargs["refspec"] = git_ref
-
-        run.run_toolbox(
-            "remote", "clone",
-            **kwargs,
-            artifact_dir_suffix="_llama_cpp",
-            force=True,
-        )
+        kwargs["version"] = version
 
-        remote_access.run_with_ansible_ssh_conf(base_work_dir, "git show -s --format='%cd%n%s%n%H' --date=format:'%y%m%d.%H%M' > ramalama-commit.info",
-                                                chdir=dest)
+    run.run_toolbox(
+        "remote", "clone",
+        **kwargs,
+        artifact_dir_suffix="_llama_cpp",
+        force=True,
+    )
+
+    remote_access.run_with_ansible_ssh_conf(base_work_dir, "git show -s --format='%cd%n%s%n%H' --date=format:'%y%m%d.%H%M' > ramalama-commit.info",
+                                            chdir=dest)
 
 
 def build_container_image(base_work_dir, ramalama_path):
@@ -100,6 +81,7 @@ def build_container_image(base_work_dir, ramalama_path):
 
     chdir = ramalama_path.parent.parent
 
+    logging.info("Building the ramalama image ...")
     with env.NextArtifactDir(f"build_ramalama_{image_name}_image"):
         cmd = f"env PATH=$PATH:{podman_mod.get_podman_binary(base_work_dir).parent}"
         cmd += f" time ./container_build.sh build {image_name}"
@@ -111,6 +93,10 @@ def build_container_image(base_work_dir, ramalama_path):
             LLAMA_CPP_REPO=config.project.get_config("prepare.llama_cpp.source.repo.url"),
         )
 
+        if config.project.get_config("prepare.ramalama.build_image.debug"):
+            extra_env["RAMALAMA_IMAGE_BUILD_DEBUG"] = "y"
+            extra_env["RAMALAMA_IMAGE_INCLUDE_DEBUG"] = "y"
+
         ret = remote_access.run_with_ansible_ssh_conf(base_work_dir, cmd,
                                                       extra_env=extra_env,
                                                       chdir=chdir, check=False, capture_stdout=True)
@@ -125,11 +111,10 @@ def build_container_image(base_work_dir, ramalama_path):
 
 
 def prepare_binary(base_work_dir, platform):
-    ramalama_path, dest, (version, git_ref) = _get_binary_path(base_work_dir, platform)
-    system_file = dest.name
+    ramalama_path, dest, version = _get_binary_path(base_work_dir, platform)
 
-    if git_ref or not remote_access.exists(ramalama_path):
-        download_ramalama(base_work_dir, dest, version, git_ref)
+    if version.startswith("pr-") or not remote_access.exists(ramalama_path):
+        download_ramalama(base_work_dir, dest, version)
     else:
         logging.info(f"ramalama {platform.name} already exists, not downloading it.")
 
@@ -138,11 +123,17 @@ def prepare_binary(base_work_dir, platform):
         image_name = build_container_image(base_work_dir, ramalama_path)
 
         if config.project.get_config("prepare.ramalama.build_image.publish.enabled"):
-            version = config.project.get_config("prepare.llama_cpp.source.repo.version")
             podman_mod.login(base_work_dir, "prepare.ramalama.build_image.publish.credentials")
+
+            version = config.project.get_config("prepare.llama_cpp.source.repo.version")
+
             dest_image_name = image_name.partition(":")[0] + f":{version}"
-            dest_image_latest = image_name.partition(":")[0] + f":latest"
-            logging.info(f"Pushing the image to {dest_image_name} and latest")
+            dest_image_latest = image_name.partition(":")[0] + ":latest"
+            if config.project.get_config("prepare.ramalama.build_image.debug"):
+                dest_image_latest = dest_image_latest.removesuffix(":latest") + ":debug"
+                dest_image_name += "-debug"
+
+            logging.info(f"Pushing the image to {dest_image_name} and {dest_image_latest}")
 
             podman_mod.push_image(base_work_dir, image_name, dest_image_name)
             podman_mod.push_image(base_work_dir, image_name, dest_image_latest)
@@ -186,13 +177,12 @@ def stop_server(base_work_dir, ramalama_path):
 def run_model(base_work_dir, platform, ramalama_path, model, unload=False):
     inference_server_port = config.project.get_config("test.inference_server.port")
 
-    if config.project.get_config("prepare.ramalama.repo.git_ref"):
-        commit_date_cmd = remote_access.run_with_ansible_ssh_conf(base_work_dir, f"cat ramalama-commit.info", chdir=ramalama_path.parent.parent, check=False, capture_stdout=True)
-        if commit_date_cmd.returncode != 0:
-            logging.warning("Couldn't find the Ramalama commit info file ...")
-        else:
-            logging.warning(f"Ramalama commit info: {commit_date_cmd.stdout}")
-            (env.ARTIFACT_DIR / "ramalama-commit.info").write_text(commit_date_cmd.stdout)
+    commit_date_cmd = remote_access.run_with_ansible_ssh_conf(base_work_dir, f"cat ramalama-commit.info", chdir=ramalama_path.parent.parent, check=False, capture_stdout=True)
+    if commit_date_cmd.returncode != 0:
+        logging.warning("Couldn't find the Ramalama commit info file ...")
+    else:
+        logging.warning(f"Ramalama commit info: {commit_date_cmd.stdout}")
+        (env.ARTIFACT_DIR / "ramalama-commit.info").write_text(commit_date_cmd.stdout)
 
     artifact_dir_suffix=None
     if unload:

projects/mac_ai/testing/remote_access.py

@@ -82,7 +82,7 @@ def prepare():
 
 def run_with_ansible_ssh_conf(
         base_work_dir, cmd,
-        extra_env={},
+        extra_env=None,
         check=True,
         capture_stdout=False,
         capture_stderr=False,
@@ -92,6 +92,9 @@ def run_with_ansible_ssh_conf(
         decode_stdout=True,
         decode_stderr=True,
 ):
+    if extra_env is None:
+        extra_env = {}
+
     run_kwargs = dict(
         log_command=False,
         check=check,