[mac-ai] Work on a releasing mechanism (#773)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added options for building and publishing remoting tarballs and
container images, with enhanced configuration for image registries and
publishing credentials.
- Introduced new script to automate Podman machine setup for remoting
scenarios on macOS.
- Added new test configuration presets for remoting, publishing, and
testing scenarios.
- Added new remote image login and push capabilities for container
management.

- **Improvements**
- Enhanced flexibility for command execution and remote access by
allowing control over output decoding and secret handling.
- Modularized and improved logic for cloning, building, and patching
source code and dependencies, ensuring fresh builds and consistent
environment setup.
- Updated preparation steps to always perform forced cloning and builds
for consistency.
- Improved platform parsing and validation for better error handling and
clarity.
- Refined remoting environment checks to activate only when remoting is
in use.
- Made container image versioning more configurable with build
arguments.
- Added control over decoding subprocess output to better handle raw or
decoded output as needed.
- Enhanced plotting logic to better handle missing or empty benchmark
data.
- Added automated creation of a comprehensive remoting release tarball
with detailed setup instructions and metadata.

- **Bug Fixes**
- Fixed platform flavor assignment to prevent errors when parsing
incomplete platform strings.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: kpouget

projects/core/library/run.py

@@ -102,7 +102,7 @@ def run_toolbox(group, command, artifact_dir_suffix=None, run_kwargs=None, mute_
     return run(f'{cmd_env} ./run_toolbox.py {group} {command} {_dict_to_run_toolbox_args(kwargs)}', **run_kwargs)
 
 
-def run(command, capture_stdout=False, capture_stderr=False, check=True, protect_shell=True, cwd=None, stdin_file=None, log_command=True):
+def run(command, capture_stdout=False, capture_stderr=False, check=True, protect_shell=True, cwd=None, stdin_file=None, log_command=True, decode_stdout=True, decode_stderr=True):
     if log_command:
         logging.info(f"run: {command}")
 
@@ -124,8 +124,8 @@ def run(command, capture_stdout=False, capture_stderr=False, check=True, protect
 
     proc = subprocess.run(command, **args)
 
-    if capture_stdout: proc.stdout = proc.stdout.decode("utf8")
-    if capture_stderr: proc.stderr = proc.stderr.decode("utf8")
+    if capture_stdout and decode_stdout: proc.stdout = proc.stdout.decode("utf8")
+    if capture_stderr and decode_stderr: proc.stderr = proc.stderr.decode("utf8")
 
     return proc
 

projects/mac_ai/testing/config.yaml

@@ -87,15 +87,38 @@ ci_presets:
   # ---
 
   remoting:
-    test.platform:
-    - podman/llama_cpp/remoting
-    - macos/llama_cpp/metal
-    - podman/llama_cpp/vulkan
     prepare.llama_cpp.source.repo.url: https://github.com/kpouget/llama.cpp
     prepare.llama_cpp.source.repo.version: main
     prepare.podman.machine.remoting_env.enabled: true
     prepare.virglrenderer.enabled: true
 
+  remoting_publish:
+    extends: [remoting]
+    prepare.remoting.publish: true
+    prepare.llama_cpp.source.repo.version: b5709-v250627
+    prepare.virglrenderer.repo.branch: v25-06-18
+    prepare.ramalama.build_image.enabled: "remoting"
+    prepare.ramalama.build_image.publish.enabled: true
+    prepare.ramalama.build_image.registry_path: quay.io/crcont
+    prepare.ramalama.build_image.name: remoting
+    prepare.ramalama.repo.url: https://github.com/kpouget/ramalama
+    prepare.ramalama.repo.git_ref: v0.9.3-remoting-rc5
+    prepare.ramalama.repo.version: null
+
+    test.platform:
+    - podman/ramalama/remoting
+    - podman/llama_cpp/remoting
+    - macos/llama_cpp/metal
+    - macos/llama_cpp/vulkan
+    # - podman/llama_cpp/vulkan # not working today
+
+  remoting_testing:
+    extends: [remoting]
+    test.platform:
+    - podman/llama_cpp/remoting
+    - macos/llama_cpp/metal
+    - podman/llama_cpp/vulkan
+
   llama_cpp:
     test.platform:
     - macos/llama_cpp/metal
@@ -192,6 +215,7 @@ secrets:
   username: mac_ai__username.mac-m4
   base_work_dir: mac_ai__base_work_dir.mac-m4
   opensearch_instances: opensearch.yaml
+  image_registry: quay-push.yaml
 
 remote_host:
   run_locally: false
@@ -290,7 +314,10 @@ prepare:
     build_image:
       enabled: false
       name: ramalama
-
+      registry_path: localhost
+      publish:
+        enabled: false
+        credentials: "*[email protected]_registry"
   brew:
     install_dependencies: false
     capture_dependencies: false
@@ -366,6 +393,9 @@ prepare:
           APIR_LLAMA_CPP_GGML_LIBRARY_REG: ggml_backend_metal_reg
           APIR_LLAMA_CPP_GGML_LIBRARY_INIT: ggml_backend_metal_init
 
+  remoting:
+    publish: false
+
 test:
   platform: *all_platforms
 
@@ -505,3 +535,4 @@ __platform_check:
 
   flavors:
     llama_cpp: [metal, vulkan, upstream_bin, kompute, remoting]
+    ramalama: [remoting, null]

projects/mac_ai/testing/images/llama_cpp.containerfile

@@ -2,13 +2,14 @@ FROM quay.io/centos/centos:10 AS builder
 USER 0
 
 ARG BUILD_FLAVOR="default"
+ARG MESA_VERSION=24.2.8-104
 
 RUN if [[ ${BUILD_FLAVOR} != remoting ]]; then \
     dnf install -y python3-dnf-plugin-versionlock && \
     dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     dnf copr enable -y slp/mesa-krunkit epel-9-aarch64 && \
-    dnf install -y mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \
-    dnf versionlock  mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \
+    dnf install -y mesa-vulkan-drivers-${MESA_VERSION}.el9.aarch64 && \
+    dnf versionlock  mesa-vulkan-drivers-${MESA_VERSION}.el9.aarch64 && \
     dnf install -y git cmake gcc gcc-c++ vulkan-loader-devel vulkan-tools fmt-devel && \
     dnf copr enable -y jeffmaury/shaderc epel-9-aarch64 && \
     dnf install -y glslc && \

projects/mac_ai/testing/podman.py

@@ -1,6 +1,7 @@
 import os
 import pathlib
 import logging
+import yaml
 
 from projects.core.library import env, config, run, configure_logging, export
 import remote_access, utils
@@ -214,3 +215,19 @@ def get_exec_command_prefix():
     podman_cmd = get_podman_command()
 
     return f"{podman_cmd} exec -it {container_name}"
+
+
+def login(base_work_dir, credentials_key):
+    creds_str = config.project.get_config(credentials_key, handled_secretly=True)
+    creds = yaml.safe_load(creds_str)
+    podman_bin = get_podman_binary(base_work_dir)
+
+    cmd = f"{podman_bin} login --username '{creds['login']}' --password '{creds['password']}' '{creds['server']}'"
+    remote_access.run_with_ansible_ssh_conf(base_work_dir, cmd, handled_secretly=True)
+
+
+def push_image(base_work_dir, local_name, remote_name):
+    podman_bin = get_podman_binary(base_work_dir)
+    cmd = f"{podman_bin} push {local_name} {remote_name}"
+
+    remote_access.run_with_ansible_ssh_conf(base_work_dir, cmd)

projects/mac_ai/testing/podman_machine.py

@@ -56,7 +56,7 @@ def start(base_work_dir, use_remoting=None):
 
     ret = _run(base_work_dir, f"start {name}", env, print_cmd=True)
 
-    if config.project.get_config("prepare.podman.machine.remoting_env.enabled"):
+    if use_remoting and config.project.get_config("prepare.podman.machine.remoting_env.enabled"):
         if not config.project.get_config("prepare.virglrenderer.enabled"):
             logging.warning("The custom virglrenderer isn't enabled ...")
 

projects/mac_ai/testing/prepare_llama_cpp.py

@@ -156,40 +156,47 @@ def prepare_from_release(base_work_dir, platform):
     return llama_cpp_path
 
 
-def prepare_from_source(base_work_dir, platform):
-    version = config.project.get_config("prepare.llama_cpp.source.repo.version")
-
+def get_source_dir(base_work_dir):
+    version = config.project.get_config("prepare.llama_cpp.source.repo.version", print=False)
     dirname = "llama.cpp-"
     if version.startswith("pr-"):
         dirname += version
     else:
         dirname += f"tag-{version}"
 
-    dest = base_work_dir / "llama_cpp" / dirname
+    return base_work_dir / "llama_cpp" / dirname
 
-    if not remote_access.exists(dest):
-        repo_url = config.project.get_config("prepare.llama_cpp.source.repo.url")
 
-        kwargs = dict(
-            repo_url=repo_url,
-            dest=dest,
-        )
+def prepare_from_source(base_work_dir, platform):
+    version = config.project.get_config("prepare.llama_cpp.source.repo.version")
 
-        if version.startswith("pr-"):
-            pr_number = version.removeprefix("pr-")
-            kwargs["refspec"] = f"refs/pull/{pr_number}/head"
-        else:
-            kwargs["version"] = version
+    dest = get_source_dir(base_work_dir)
 
-        run.run_toolbox(
-            "remote", "clone",
-            **kwargs,
-            artifact_dir_suffix="_llama_cpp",
-        )
+    # don't check if already exists, always build it
+
+    repo_url = config.project.get_config("prepare.llama_cpp.source.repo.url")
+
+    kwargs = dict(
+        repo_url=repo_url,
+        dest=dest,
+    )
+
+    if version.startswith("pr-"):
+        pr_number = version.removeprefix("pr-")
+        kwargs["refspec"] = f"refs/pull/{pr_number}/head"
+    else:
+        kwargs["version"] = version
+
+    run.run_toolbox(
+        "remote", "clone",
+        **kwargs,
+        force=True,
+        artifact_dir_suffix="_llama_cpp",
+    )
 
-        # for the Kompute build
-        cmd = f"sed -i.bu s/-Werror//g {dest}/ggml/src/ggml-kompute/kompute/CMakeLists.txt"
-        remote_access.run_with_ansible_ssh_conf(base_work_dir, cmd)
+    # for the Kompute build
+    cmd = f"sed -i.bu s/-Werror//g {dest}/ggml/src/ggml-kompute/kompute/CMakeLists.txt"
+    remote_access.run_with_ansible_ssh_conf(base_work_dir, cmd)
 
     src_dir = dest
     cmake_parallel = config.project.get_config("prepare.llama_cpp.source.cmake.parallel")

projects/mac_ai/testing/prepare_mac_ai.py

@@ -5,8 +5,8 @@
 from projects.core.library import env, config, run, configure_logging, export
 from projects.matrix_benchmarking.library import visualize
 
-import prepare_llama_cpp, utils, remote_access, podman_machine, brew, podman, prepare_virglrenderer
-import llama_cpp, ollama, ramalama
+import utils, remote_access, podman_machine, brew, podman, prepare_virglrenderer, prepare_release
+import prepare_llama_cpp, llama_cpp, ollama, ramalama
 
 TESTING_THIS_DIR = pathlib.Path(__file__).absolute().parent
 CRC_MAC_AI_SECRET_PATH = pathlib.Path(os.environ.get("CRC_MAC_AI_SECRET_PATH", "/env/CRC_MAC_AI_SECRET_PATH/not_set"))
@@ -26,6 +26,7 @@
 
 REMOTING_FRONTEND_PLATFORM = "podman/llama_cpp/remoting"
 REMOTING_BACKEND_PLATFORM = "macos/llama_cpp/remoting"
+RAMALAMA_REMOTING_PLATFORM = "podman/ramalama/remoting"
 
 def cleanup():
     base_work_dir = remote_access.prepare()
@@ -92,7 +93,10 @@ def cleanup():
 
 def prepare():
     base_work_dir = remote_access.prepare()
+
     if not config.project.get_config("prepare.prepare_only_inference_server"):
+        run.run_toolbox("mac_ai", "remote_capture_system_state")
+
         if config.project.get_config("prepare.podman.repo.enabled"):
             podman.prepare_from_gh_binary(base_work_dir)
             podman.prepare_gv_from_gh_binary(base_work_dir)
@@ -173,6 +177,16 @@ def prepare_binary():
     if config.project.get_config("prepare.podman.machine.enabled"):
         podman_machine.configure_and_start(base_work_dir, force_restart=False)
 
+    if config.project.get_config("prepare.remoting.publish"):
+        if config.project.get_config("exec_list.pre_cleanup_ci") is False:
+            raise ValueError("Cannot publish the remoting libraries if not preparing from a clean environment")
+        if not config.project.get_config("prepare.virglrenderer.enabled"):
+            raise ValueError("Cannot publish the remoting libraries if building virglrenderer isn't enabled")
+        if "podman/llama_cpp/remoting" not in platforms_to_build_str:
+            raise ValueError("Cannot publish the remoting libraries if podman/llama_cpp/remoting isn't built")
+
+        prepare_release.create_remoting_tarball(base_work_dir)
+
     return 0