Add support for multi-nodes on builders

docs/docs_utils.py

@@ -206,6 +206,16 @@ def generate_code_demo_builders() -> str:
         to_evaluate = [
             "builder.name",
             "builder.driver",
+            "builder.last_activity",
+            "builder.dynamic",
+            "builder.nodes[0].name",
+            "builder.nodes[0].endpoint",
+            "builder.nodes[0].flags",
+            "builder.nodes[0].status",
+            "builder.nodes[0].version",
+            "builder.nodes[0].ids",
+            "builder.nodes[0].platforms",
+            "builder.nodes[0].labels",
         ]
 
         for i, attribute_access in enumerate(to_evaluate):

python_on_whales/client_config.py

@@ -206,7 +206,6 @@ def _needs_reload(self) -> bool:
         )
 
     def reload(self):
-        print("fetching and parsing inspect result", self._immutable_id)
         self._set_inspect_result(
             self._fetch_and_parse_inspect_result(self._immutable_id)
         )

python_on_whales/components/buildx/cli_wrapper.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import datetime as dt
 import json
 import tempfile
 from enum import Enum
@@ -24,7 +25,7 @@
     ReloadableObject,
 )
 from python_on_whales.components.buildx.imagetools.cli_wrapper import ImagetoolsCLI
-from python_on_whales.components.buildx.models import BuilderInspectResult
+from python_on_whales.components.buildx.models import BuilderInspectResult, BuilderNode
 from python_on_whales.utils import (
     ValidPath,
     format_mapping_for_cli,
@@ -43,7 +44,7 @@ class Builder(ReloadableObject):
     def __init__(
         self,
         client_config: ClientConfig,
-        reference: Optional[str],
+        reference: str,
         is_immutable_id=False,
     ):
         super().__init__(client_config, "name", reference, is_immutable_id)
@@ -54,30 +55,39 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.remove()
 
-    def _fetch_and_parse_inspect_result(
-        self, reference: Optional[str]
-    ) -> BuilderInspectResult:
-        full_cmd = self.docker_cmd + ["buildx", "inspect"]
-        if reference is not None:
-            full_cmd.append(reference)
+    def _fetch_and_parse_inspect_result(self, reference: str) -> BuilderInspectResult:
+        full_cmd = self.docker_cmd + ["buildx", "ls", "--format", "{{json . }}"]
         inspect_str = run(full_cmd)
-        return BuilderInspectResult.from_str(inspect_str)
+        lines = inspect_str.splitlines()
+        for line in lines:
+            builder_inspect_result = BuilderInspectResult(**json.loads(line))
+            if builder_inspect_result.name == reference:
+                return builder_inspect_result
+        raise ValueError(
+            f"Could not find a builder with the name {reference}, this should "
+            f"never happen unless you are quickly creating and deleting builders. "
+            f"Please open an issue at https://github.com/gabrieldemarmiesse/python-on-whales/issues"
+        )
 
     @property
-    def name(self) -> str:
+    def name(self) -> str | None:
         return self._get_immutable_id()
 
     @property
-    def driver(self) -> str:
+    def driver(self) -> str | None:
         return self._get_inspect_result().driver
 
     @property
-    def status(self) -> str:
-        return self._get_inspect_result().status
+    def last_activity(self) -> dt.datetime | None:
+        return self._get_inspect_result().last_activity
 
     @property
-    def platforms(self) -> List[str]:
-        return self._get_inspect_result().platforms
+    def dynamic(self) -> bool | None:
+        return self._get_inspect_result().dynamic
+
+    @property
+    def nodes(self) -> List[BuilderNode] | None:
+        return self._get_inspect_result().nodes
 
     def __repr__(self):
         return f"python_on_whales.Builder(name='{self.name}', driver='{self.driver}')"
@@ -466,6 +476,7 @@ def create(
         driver_options: Dict[str, str] = {},
         name: Optional[str] = None,
         use: bool = False,
+        append: bool = False,
     ) -> Builder:
         """Create a new builder instance
 
@@ -481,6 +492,8 @@ def create(
                 e.g `driver_options=dict(network="host")`
             name: Builder instance name
             use: Set the current builder instance to this builder
+            append: Append a node to the current builder instance, in this case
+                `name` must be provided.
 
         # Returns
             A `python_on_whales.Builder` object.
@@ -490,6 +503,7 @@ def create(
         full_cmd.add_flag("--bootstrap", bootstrap)
         full_cmd.add_simple_arg("--buildkitd-flags", buildkitd_flags)
         full_cmd.add_simple_arg("--config", config)
+        full_cmd.add_flag("--append", append)
         if platforms is not None:
             full_cmd += ["--platform", ",".join(platforms)]
         full_cmd.add_simple_arg("--driver", driver)
@@ -523,29 +537,40 @@ def inspect(
         # Returns
             A `python_on_whales.Builder` object.
         """
-        if bootstrap:
-            full_cmd = self.docker_cmd + ["buildx", "inspect"]
-            if x is not None:
-                full_cmd.append(x)
-            full_cmd.add_flag("--bootstrap", bootstrap)
-            run(full_cmd)
-        return Builder(self.client_config, x, is_immutable_id=False)
+        # Sadly, docker buildx inspect has no json support,
+        # so, it's ugly, but we grab the name from the first line and
+        # then filter "docker buildx ls".
+        full_cmd = self.docker_cmd + ["buildx", "inspect"]
+        if x is not None:
+            full_cmd.append(x)
+        full_cmd.add_flag("--bootstrap", bootstrap)
+        output = run(full_cmd)
+        name = output.splitlines()[0].split(":")[1].strip()
+        for builder in self.list():
+            if builder.name == name:
+                return builder
+        raise ValueError(
+            f"Could not find a builder with the name {name}, this should "
+            f"never happen unless you are quickly creating and deleting builders. "
+            f"Please open an issue at https://github.com/gabrieldemarmiesse/python-on-whales/issues."
+        )
 
     def list(self) -> List[Builder]:
         """Returns the list of `python_on_whales.Builder` available."""
-        full_cmd = self.docker_cmd + ["buildx", "ls"]
+        full_cmd = self.docker_cmd + ["buildx", "ls", "--format", "{{json . }}"]
         output = run(full_cmd)
         lines = output.splitlines()
-        # the first line have the headers
-        lines = lines[1:]
-        # if the line starts by a " ", it's not a builder, it's a node
-        lines = list(filter(lambda x: not x.startswith(" "), lines))
-        builders_names = [x.split(" ")[0] for x in lines]
-        # in buildx 0.13.0, the "*" is added to the name, without whitespace
-        builders_names = [removesuffix(x, "*") for x in builders_names]
-        return [
-            Builder(self.client_config, x, is_immutable_id=True) for x in builders_names
-        ]
+        lines = set(lines)
+
+        inpect_results = [BuilderInspectResult(**json.loads(line)) for line in lines]
+        result = []
+        for inspect_result in inpect_results:
+            builder = Builder(
+                self.client_config, inspect_result.name, is_immutable_id=True
+            )
+            builder._set_inspect_result(inspect_result)
+            result.append(builder)
+        return result
 
     @overload
     def prune(

python_on_whales/components/buildx/models.py

@@ -1,30 +1,28 @@
 from __future__ import annotations
 
-from dataclasses import dataclass, field
-from typing import List
+import datetime as dt
+from typing import Dict, List, Optional
 
+from pydantic import Field
+from typing_extensions import Annotated
 
-@dataclass
-class BuilderInspectResult:
-    name: str
-    driver: str
-    status: str
-    platforms: List[str] = field(default_factory=lambda: [])
+from python_on_whales.utils import DockerCamelModel
 
-    @classmethod
-    def from_str(cls, string: str) -> BuilderInspectResult:
-        string = string.strip()
-        result_dict = {}
-        for line in string.splitlines():
-            if line.startswith("Name:") and "name" not in result_dict:
-                result_dict["name"] = line.split(":")[1].strip()
-            if line.startswith("Driver:"):
-                result_dict["driver"] = line.split(":")[1].strip()
-            if line.startswith("Status:"):
-                result_dict["status"] = line.split(":")[1].strip()
-            if line.startswith("Platforms:"):
-                platforms = line.split(":")[1].strip()
-                if platforms:
-                    result_dict["platforms"] = platforms.split(", ")
 
-        return cls(**result_dict)
+class BuilderNode(DockerCamelModel):
+    name: Optional[str] = None
+    endpoint: Optional[str] = None
+    flags: Optional[List[str]] = None
+    status: Optional[str] = None
+    version: Optional[str] = None
+    ids: Annotated[Optional[List[str]], Field(alias="IDs")] = None
+    platforms: Optional[List[str]] = None
+    labels: Optional[Dict[str, str]] = None
+
+
+class BuilderInspectResult(DockerCamelModel):
+    name: Optional[str] = None
+    driver: Optional[str] = None
+    last_activity: Optional[dt.datetime] = None
+    dynamic: Optional[bool] = None
+    nodes: Optional[List[BuilderNode]] = None

scripts/download-docker-plugins.sh

@@ -5,5 +5,5 @@ set -e
 mkdir -p ~/.docker/cli-plugins/
 wget -q https://github.com/docker/compose/releases/download/v2.26.0/docker-compose-linux-x86_64 -O ~/.docker/cli-plugins/docker-compose
 chmod +x ~/.docker/cli-plugins/docker-compose
-wget -q https://github.com/docker/buildx/releases/download/v0.10.0/buildx-v0.10.0.linux-amd64 -O ~/.docker/cli-plugins/docker-buildx
+wget -q https://github.com/docker/buildx/releases/download/v0.13.0/buildx-v0.13.0.linux-amd64 -O ~/.docker/cli-plugins/docker-buildx
 chmod +x ~/.docker/cli-plugins/docker-buildx

tests/python_on_whales/components/buildx/test_buildx_cli_wrapper.py

@@ -4,7 +4,6 @@
 import pytest
 
 from python_on_whales import docker
-from python_on_whales.components.buildx.models import BuilderInspectResult
 from python_on_whales.exceptions import DockerException
 from python_on_whales.test_utils import set_cache_validity_period
 from python_on_whales.utils import PROJECT_ROOT
@@ -370,9 +369,9 @@ def test_buildx_inspect_bootstrap():
     my_builder = docker.buildx.create()
     with my_builder:
         docker.buildx.inspect(my_builder.name, bootstrap=True)
-        assert my_builder.status == "running"
+        assert my_builder.nodes[0].status == "running"
         # Must contain at least the host native platform
-        assert my_builder.platforms
+        assert my_builder.nodes[0].platforms
 
 
 def test_builder_name():
@@ -487,58 +486,19 @@ def test_buildx_create_remove():
 def test_buildx_create_bootstrap():
     my_builder = docker.buildx.create(bootstrap=True)
     with my_builder:
-        assert my_builder.status == "running"
+        assert my_builder.nodes[0].status == "running"
         # Must contain at least the host native platform
-        assert my_builder.platforms
+        assert my_builder.nodes[0].platforms
 
 
 def test_buildx_create_remove_with_platforms():
     builder = docker.buildx.create(platforms=["linux/amd64", "linux/arm64"])
 
-    assert builder.platforms == ["linux/amd64*", "linux/arm64*"]
+    assert builder.nodes[0].platforms == ["linux/amd64", "linux/arm64"]
 
     docker.buildx.remove(builder)
 
 
-some_builder_info = """
-Name:   blissful_swartz
-Driver: docker-container
-
-Nodes:
-Name:      blissful_swartz0
-Endpoint:  unix:///var/run/docker.sock
-Status:    inactive
-Platforms:
-"""
-
-some_builder_info_with_platforms = """
-Name:   blissful_swartz
-Driver: docker-container
-
-Nodes:
-Name:      blissful_swartz0
-Endpoint:  unix:///var/run/docker.sock
-Status:    running
-Platforms: linux/amd64, linux/arm64
-"""
-
-
-def test_builder_inspect_result_from_string():
-    a = BuilderInspectResult.from_str(some_builder_info)
-    assert a.name == "blissful_swartz"
-    assert a.driver == "docker-container"
-    assert a.status == "inactive"
-    assert a.platforms == []
-
-
-def test_builder_inspect_result_platforms_from_string():
-    a = BuilderInspectResult.from_str(some_builder_info_with_platforms)
-    assert a.name == "blissful_swartz"
-    assert a.driver == "docker-container"
-    assert a.status == "running"
-    assert a.platforms == ["linux/amd64", "linux/arm64"]
-
-
 bake_test_dir = PROJECT_ROOT / "tests/python_on_whales/components/bake_tests"
 bake_file = bake_test_dir / "docker-bake.hcl"