support multiple osd per node

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Vscode
+.vscode/

ceph_devstack/ceph_devstack.te

@@ -28,6 +28,7 @@ require {
 
 	type fixed_disk_device_t;
 	class blk_file setattr;
+	class blk_file mounton;
 
 	type fs_t;
 
@@ -68,6 +69,10 @@ require {
 
 	class bpf prog_load;
 	class bpf map_create;
+
+	type fuse_device_t;
+
+	type tun_tap_device_t;
 }
 
 #============= container_init_t ==============
@@ -106,3 +111,6 @@ allow container_init_t system_map_t:file mounton;
 allow container_init_t mtrr_device_t:file mounton;
 allow container_init_t self:bpf prog_load;
 allow container_init_t self:bpf map_create;
+allow container_init_t fuse_device_t:chr_file mounton;
+allow container_init_t fixed_disk_device_t:blk_file mounton;
+allow container_init_t tun_tap_device_t:chr_file mounton;

ceph_devstack/resources/ceph/containers.py

@@ -21,7 +21,7 @@ class Postgres(Container):
         "--network",
         "ceph-devstack",
         "-p",
-        "5432:5432",
+        "5431:5432",
         "--health-cmd",
         "CMD pg_isready -q -d paddles -U admin",
         "--health-interval",
@@ -170,53 +170,6 @@ class TestNode(Container):
         "AUDIT_WRITE",
         "AUDIT_CONTROL",
     ]
-    create_cmd = [
-        "podman",
-        "container",
-        "create",
-        "--rm",
-        "-i",
-        "--network",
-        "ceph-devstack",
-        "--systemd=always",
-        "--cgroupns=host",
-        "--secret",
-        "id_rsa.pub",
-        "-p",
-        "22",
-        "--cap-add",
-        ",".join(capabilities),
-        "--security-opt",
-        "unmask=/sys/dev/block",
-        "-v",
-        "/sys/dev/block:/sys/dev/block",
-        "-v",
-        "/sys/fs/cgroup:/sys/fs/cgroup",
-        "-v",
-        "/dev/fuse:/dev/fuse",
-        "-v",
-        "/dev/disk:/dev/disk",
-        # cephadm tries to access these DMI-related files, and by default they
-        # have 600 permissions on the host. It appears to be ok if they are
-        # empty, though.
-        # The below was bizarrely causing this error message:
-        # No such file or directory: OCI runtime attempted to invoke a command that was
-        # not found
-        # That was causing the container to fail to start up.
-        # "-v",
-        # "/dev/null:/sys/class/dmi/id/board_serial",
-        "-v",
-        "/dev/null:/sys/class/dmi/id/chassis_serial",
-        "-v",
-        "/dev/null:/sys/class/dmi/id/product_serial",
-        "--device",
-        "/dev/net/tun",
-        "--device",
-        "{loop_dev_name}",
-        "--name",
-        "{name}",
-        "{image}",
-    ]
     env_vars = {
         "SSH_PUBKEY": "",
         "CEPH_VOLUME_ALLOW_LOOP_DEVICES": "true",
@@ -231,42 +184,107 @@ def __init__(self, name: str = ""):
         else:
             self.loop_img_name += str(self.loop_index)
         self.loop_dev_name = f"/dev/loop{self.loop_index}"
+        self.osd_count = int(
+            os.getenv(
+                "CEPH_OSD_COUNT", config["containers"]["testnode"].get("osd_count", 1)
+            )
+        )
+        self.devices = [self.device_name(i) for i in range(self.osd_count)]
 
     @property
     def loop_img_dir(self):
         return (Path(config["data_dir"]) / "disk_images").expanduser()
 
+    @property
+    def create_cmd(self):
+        return [
+            "podman",
+            "container",
+            "create",
+            "--rm",
+            "-i",
+            "--network",
+            "ceph-devstack",
+            "--systemd=always",
+            "--cgroupns=host",
+            "--secret",
+            "id_rsa.pub",
+            "-p",
+            "22",
+            "--cap-add",
+            ",".join(self.capabilities),
+            "--security-opt",
+            "unmask=/sys/dev/block",
+            "-v",
+            "/sys/dev/block:/sys/dev/block",
+            "-v",
+            "/sys/fs/cgroup:/sys/fs/cgroup",
+            "-v",
+            "/dev/fuse:/dev/fuse",
+            "-v",
+            "/dev/disk:/dev/disk",
+            # cephadm tries to access these DMI-related files, and by default they
+            # have 600 permissions on the host. It appears to be ok if they are
+            # empty, though.
+            # The below was bizarrely causing this error message:
+            # No such file or directory: OCI runtime attempted to invoke a command that was
+            # not found
+            # That was causing the container to fail to start up.
+            "-v",
+            "/dev/null:/sys/class/dmi/id/board_serial",
+            "-v",
+            "/dev/null:/sys/class/dmi/id/chassis_serial",
+            "-v",
+            "/dev/null:/sys/class/dmi/id/product_serial",
+            "--device",
+            "/dev/net/tun",
+            *[f"--device={device}" for device in self.devices],
+            "--name",
+            "{name}",
+            "{image}",
+        ]
+
     async def create(self):
         if not await self.exists():
-            await self.create_loop_device()
+            await self.create_loop_devices()
         await super().create()
 
     async def remove(self):
         await super().remove()
-        await self.remove_loop_device()
+        await self.remove_loop_devices()
 
-    async def create_loop_device(self):
+    async def create_loop_devices(self):
+        for i in range(self.osd_count):
+            await self.create_loop_device(i)
+
+    def device_name(self, index: int):
+        if self.loop_index == 0:
+            return f"/dev/loop{index}"
+        return f"{self.loop_dev_name}{index}"
+
+    async def create_loop_device(self, index: int):
         size_gb = 5
         os.makedirs(self.loop_img_dir, exist_ok=True)
         proc = await self.cmd(["lsmod", "|", "grep", "loop"])
         if proc and await proc.wait() != 0:
             await self.cmd(["sudo", "modprobe", "loop"])
-        loop_img_name = os.path.join(self.loop_img_dir, self.loop_img_name)
-        await self.remove_loop_device()
+        loop_img_name = os.path.join(self.loop_img_dir, f"{self.loop_img_name}.{index}")
+        loop_dev_name = self.device_name(index)
+        await self.remove_loop_device(index)
         await self.cmd(
             [
                 "sudo",
                 "mknod",
                 "-m700",
-                self.loop_dev_name,
+                loop_dev_name,
                 "b",
                 "7",
-                str(self.loop_index),
+                f"{self.loop_index}{index}",
             ],
             check=True,
         )
         await self.cmd(
-            ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", self.loop_dev_name],
+            ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", loop_dev_name],
             check=True,
         )
         await self.cmd(
@@ -281,17 +299,20 @@ async def create_loop_device(self):
             ],
             check=True,
         )
-        await self.cmd(
-            ["sudo", "losetup", self.loop_dev_name, loop_img_name], check=True
-        )
+        await self.cmd(["sudo", "losetup", loop_dev_name, loop_img_name], check=True)
+
+    async def remove_loop_devices(self):
+        for i in range(self.osd_count):
+            await self.remove_loop_device(i)
 
-    async def remove_loop_device(self):
-        loop_img_name = os.path.join(self.loop_img_dir, self.loop_img_name)
-        if os.path.ismount(self.loop_dev_name):
-            await self.cmd(["umount", self.loop_dev_name], check=True)
-        if host.path_exists(self.loop_dev_name):
-            await self.cmd(["sudo", "losetup", "-d", self.loop_dev_name])
-            await self.cmd(["sudo", "rm", "-f", self.loop_dev_name], check=True)
+    async def remove_loop_device(self, index: int):
+        loop_img_name = os.path.join(self.loop_img_dir, f"{self.loop_img_name}.{index}")
+        loop_dev_name = self.device_name(index)
+        if os.path.ismount(loop_dev_name):
+            await self.cmd(["umount", loop_dev_name], check=True)
+        if host.path_exists(loop_dev_name):
+            await self.cmd(["sudo", "losetup", "-d", loop_dev_name])
+            await self.cmd(["sudo", "rm", "-f", loop_dev_name], check=True)
         if host.path_exists(loop_img_name):
             os.remove(loop_img_name)
 

ceph_devstack/resources/test/fixtures/osd_config.toml

@@ -0,0 +1,2 @@
+[containers.testnode]
+osd_count = 8

ceph_devstack/resources/test/test_testnode_container.py

@@ -0,0 +1,60 @@
+import os
+from pathlib import Path
+
+import pytest
+
+from ceph_devstack import Config
+from ceph_devstack.resources.ceph.containers import TestNode
+
+
+class TestTestnodeContainer:
+    @pytest.fixture(scope="class")
+    def cls(self):
+        return TestNode
+
+    def test_osd_count_is_read_from_env_variable(self, cls: type[TestNode]):
+        os.environ["CEPH_OSD_COUNT"] = "3"
+        testnode = cls()
+
+        assert testnode.osd_count == 3
+        del os.environ["CEPH_OSD_COUNT"]
+
+    def test_config_loads_osd_count(self):
+        config = Config()
+        config.load(Path(__file__).parent.joinpath("fixtures", "osd_config.toml"))
+        assert (
+            config["containers"]["testnode"]["osd_count"] == 8
+        )  # TODO: Find a way to mock config to make assert on testnode.osd_count
+
+    def test_osd_count_defaults_to_1(self, cls: type[TestNode]):
+        testnode = cls()
+        assert testnode.osd_count == 1
+
+    def test_devices_property(self, cls: type[TestNode]):
+        testnode = cls()
+        assert len(testnode.devices) == testnode.osd_count
+        for i, device in zip(range(testnode.osd_count), testnode.devices):
+            assert testnode.device_name(i) == device
+
+    def test_device_name_returns_correct_name(self, cls: type[TestNode]):
+        testnode = cls()
+        assert testnode.device_name(0) == "/dev/loop0"
+        assert testnode.device_name(1) == "/dev/loop1"
+        assert testnode.device_name(2) == "/dev/loop2"
+
+    def test_device_name_on_multiple_testnodes(self, cls: type[TestNode]):
+        testnode = cls("testnode_1")
+
+        assert testnode.device_name(0) == "/dev/loop10"
+        assert testnode.device_name(1) == "/dev/loop11"
+        assert testnode.device_name(2) == "/dev/loop12"
+
+    def test_testnode_create_cmd_includes_devices(self, cls: type[TestNode]):
+        osd_count = 3
+        os.environ["CEPH_OSD_COUNT"] = str(osd_count)
+        testnode = cls("testnode_1")
+        cmd = testnode.create_cmd
+
+        assert "--device=/dev/loop10" in cmd
+        assert "--device=/dev/loop11" in cmd
+        assert "--device=/dev/loop12" in cmd