diff --git a/.gitignore b/.gitignore index b6e47617d..663766301 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# Vscode +.vscode/ \ No newline at end of file diff --git a/ceph_devstack/ceph_devstack.pp b/ceph_devstack/ceph_devstack.pp index ca7654a49..978a93980 100644 Binary files a/ceph_devstack/ceph_devstack.pp and b/ceph_devstack/ceph_devstack.pp differ diff --git a/ceph_devstack/ceph_devstack.te b/ceph_devstack/ceph_devstack.te index ec5deda7c..41a984f76 100644 --- a/ceph_devstack/ceph_devstack.te +++ b/ceph_devstack/ceph_devstack.te @@ -28,6 +28,7 @@ require { type fixed_disk_device_t; class blk_file setattr; + class blk_file mounton; type fs_t; @@ -68,6 +69,10 @@ require { class bpf prog_load; class bpf map_create; + + type fuse_device_t; + + type tun_tap_device_t; } #============= container_init_t ============== @@ -106,3 +111,6 @@ allow container_init_t system_map_t:file mounton; allow container_init_t mtrr_device_t:file mounton; allow container_init_t self:bpf prog_load; allow container_init_t self:bpf map_create; +allow container_init_t fuse_device_t:chr_file mounton; +allow container_init_t fixed_disk_device_t:blk_file mounton; +allow container_init_t tun_tap_device_t:chr_file mounton; diff --git a/ceph_devstack/resources/ceph/containers.py b/ceph_devstack/resources/ceph/containers.py index c36d5b781..a1cccdc06 100644 --- a/ceph_devstack/resources/ceph/containers.py +++ b/ceph_devstack/resources/ceph/containers.py @@ -21,7 +21,7 @@ class Postgres(Container): "--network", "ceph-devstack", "-p", - "5432:5432", + "5431:5432", "--health-cmd", "CMD pg_isready -q -d paddles -U admin", "--health-interval", @@ -170,53 +170,6 @@ class TestNode(Container): "AUDIT_WRITE", "AUDIT_CONTROL", ] - create_cmd = [ - "podman", - "container", - "create", - "--rm", - "-i", - "--network", - "ceph-devstack", - "--systemd=always", - "--cgroupns=host", - "--secret", - "id_rsa.pub", - "-p", - "22", - "--cap-add", - ",".join(capabilities), - "--security-opt", - "unmask=/sys/dev/block", - "-v", - "/sys/dev/block:/sys/dev/block", - "-v", - "/sys/fs/cgroup:/sys/fs/cgroup", - "-v", - "/dev/fuse:/dev/fuse", - "-v", - "/dev/disk:/dev/disk", - # cephadm tries to access these DMI-related files, and by default they - # have 600 permissions on the host. It appears to be ok if they are - # empty, though. - # The below was bizarrely causing this error message: - # No such file or directory: OCI runtime attempted to invoke a command that was - # not found - # That was causing the container to fail to start up. - # "-v", - # "/dev/null:/sys/class/dmi/id/board_serial", - "-v", - "/dev/null:/sys/class/dmi/id/chassis_serial", - "-v", - "/dev/null:/sys/class/dmi/id/product_serial", - "--device", - "/dev/net/tun", - "--device", - "{loop_dev_name}", - "--name", - "{name}", - "{image}", - ] env_vars = { "SSH_PUBKEY": "", "CEPH_VOLUME_ALLOW_LOOP_DEVICES": "true", @@ -231,42 +184,107 @@ def __init__(self, name: str = ""): else: self.loop_img_name += str(self.loop_index) self.loop_dev_name = f"/dev/loop{self.loop_index}" + self.osd_count = int( + os.getenv( + "CEPH_OSD_COUNT", config["containers"]["testnode"].get("osd_count", 1) + ) + ) + self.devices = [self.device_name(i) for i in range(self.osd_count)] @property def loop_img_dir(self): return (Path(config["data_dir"]) / "disk_images").expanduser() + @property + def create_cmd(self): + return [ + "podman", + "container", + "create", + "--rm", + "-i", + "--network", + "ceph-devstack", + "--systemd=always", + "--cgroupns=host", + "--secret", + "id_rsa.pub", + "-p", + "22", + "--cap-add", + ",".join(self.capabilities), + "--security-opt", + "unmask=/sys/dev/block", + "-v", + "/sys/dev/block:/sys/dev/block", + "-v", + "/sys/fs/cgroup:/sys/fs/cgroup", + "-v", + "/dev/fuse:/dev/fuse", + "-v", + "/dev/disk:/dev/disk", + # cephadm tries to access these DMI-related files, and by default they + # have 600 permissions on the host. It appears to be ok if they are + # empty, though. + # The below was bizarrely causing this error message: + # No such file or directory: OCI runtime attempted to invoke a command that was + # not found + # That was causing the container to fail to start up. + "-v", + "/dev/null:/sys/class/dmi/id/board_serial", + "-v", + "/dev/null:/sys/class/dmi/id/chassis_serial", + "-v", + "/dev/null:/sys/class/dmi/id/product_serial", + "--device", + "/dev/net/tun", + *[f"--device={device}" for device in self.devices], + "--name", + "{name}", + "{image}", + ] + async def create(self): if not await self.exists(): - await self.create_loop_device() + await self.create_loop_devices() await super().create() async def remove(self): await super().remove() - await self.remove_loop_device() + await self.remove_loop_devices() - async def create_loop_device(self): + async def create_loop_devices(self): + for i in range(self.osd_count): + await self.create_loop_device(i) + + def device_name(self, index: int): + if self.loop_index == 0: + return f"/dev/loop{index}" + return f"{self.loop_dev_name}{index}" + + async def create_loop_device(self, index: int): size_gb = 5 os.makedirs(self.loop_img_dir, exist_ok=True) proc = await self.cmd(["lsmod", "|", "grep", "loop"]) if proc and await proc.wait() != 0: await self.cmd(["sudo", "modprobe", "loop"]) - loop_img_name = os.path.join(self.loop_img_dir, self.loop_img_name) - await self.remove_loop_device() + loop_img_name = os.path.join(self.loop_img_dir, f"{self.loop_img_name}.{index}") + loop_dev_name = self.device_name(index) + await self.remove_loop_device(index) await self.cmd( [ "sudo", "mknod", "-m700", - self.loop_dev_name, + loop_dev_name, "b", "7", - str(self.loop_index), + f"{self.loop_index}{index}", ], check=True, ) await self.cmd( - ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", self.loop_dev_name], + ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", loop_dev_name], check=True, ) await self.cmd( @@ -281,17 +299,20 @@ async def create_loop_device(self): ], check=True, ) - await self.cmd( - ["sudo", "losetup", self.loop_dev_name, loop_img_name], check=True - ) + await self.cmd(["sudo", "losetup", loop_dev_name, loop_img_name], check=True) + + async def remove_loop_devices(self): + for i in range(self.osd_count): + await self.remove_loop_device(i) - async def remove_loop_device(self): - loop_img_name = os.path.join(self.loop_img_dir, self.loop_img_name) - if os.path.ismount(self.loop_dev_name): - await self.cmd(["umount", self.loop_dev_name], check=True) - if host.path_exists(self.loop_dev_name): - await self.cmd(["sudo", "losetup", "-d", self.loop_dev_name]) - await self.cmd(["sudo", "rm", "-f", self.loop_dev_name], check=True) + async def remove_loop_device(self, index: int): + loop_img_name = os.path.join(self.loop_img_dir, f"{self.loop_img_name}.{index}") + loop_dev_name = self.device_name(index) + if os.path.ismount(loop_dev_name): + await self.cmd(["umount", loop_dev_name], check=True) + if host.path_exists(loop_dev_name): + await self.cmd(["sudo", "losetup", "-d", loop_dev_name]) + await self.cmd(["sudo", "rm", "-f", loop_dev_name], check=True) if host.path_exists(loop_img_name): os.remove(loop_img_name) diff --git a/ceph_devstack/resources/test/fixtures/osd_config.toml b/ceph_devstack/resources/test/fixtures/osd_config.toml new file mode 100644 index 000000000..67fba5166 --- /dev/null +++ b/ceph_devstack/resources/test/fixtures/osd_config.toml @@ -0,0 +1,2 @@ +[containers.testnode] +osd_count = 8 diff --git a/ceph_devstack/resources/test/test_testnode_container.py b/ceph_devstack/resources/test/test_testnode_container.py new file mode 100644 index 000000000..8d6e46b11 --- /dev/null +++ b/ceph_devstack/resources/test/test_testnode_container.py @@ -0,0 +1,60 @@ +import os +from pathlib import Path + +import pytest + +from ceph_devstack import Config +from ceph_devstack.resources.ceph.containers import TestNode + + +class TestTestnodeContainer: + @pytest.fixture(scope="class") + def cls(self): + return TestNode + + def test_osd_count_is_read_from_env_variable(self, cls: type[TestNode]): + os.environ["CEPH_OSD_COUNT"] = "3" + testnode = cls() + + assert testnode.osd_count == 3 + del os.environ["CEPH_OSD_COUNT"] + + def test_config_loads_osd_count(self): + config = Config() + config.load(Path(__file__).parent.joinpath("fixtures", "osd_config.toml")) + assert ( + config["containers"]["testnode"]["osd_count"] == 8 + ) # TODO: Find a way to mock config to make assert on testnode.osd_count + + def test_osd_count_defaults_to_1(self, cls: type[TestNode]): + testnode = cls() + assert testnode.osd_count == 1 + + def test_devices_property(self, cls: type[TestNode]): + testnode = cls() + assert len(testnode.devices) == testnode.osd_count + for i, device in zip(range(testnode.osd_count), testnode.devices): + assert testnode.device_name(i) == device + + def test_device_name_returns_correct_name(self, cls: type[TestNode]): + testnode = cls() + assert testnode.device_name(0) == "/dev/loop0" + assert testnode.device_name(1) == "/dev/loop1" + assert testnode.device_name(2) == "/dev/loop2" + + def test_device_name_on_multiple_testnodes(self, cls: type[TestNode]): + testnode = cls("testnode_1") + + assert testnode.device_name(0) == "/dev/loop10" + assert testnode.device_name(1) == "/dev/loop11" + assert testnode.device_name(2) == "/dev/loop12" + + def test_testnode_create_cmd_includes_devices(self, cls: type[TestNode]): + osd_count = 3 + os.environ["CEPH_OSD_COUNT"] = str(osd_count) + testnode = cls("testnode_1") + cmd = testnode.create_cmd + + assert "--device=/dev/loop10" in cmd + assert "--device=/dev/loop11" in cmd + assert "--device=/dev/loop12" in cmd