Merge pull request ceph#64323 from guits/cv-seastore

ceph-volume: add seastore OSDs support

Reviewed-by: Adam King <[email protected]>
Reviewed-by: Matan Breizman <[email protected]>
Reviewed-by: Yingxin Cheng <[email protected]>

Author: adk3798

src/ceph-volume/ceph_volume/activate/main.py

@@ -3,8 +3,8 @@
 import argparse
 
 from ceph_volume import terminal
-from ceph_volume.objectstore.lvmbluestore import LvmBlueStore as LVMActivate
-from ceph_volume.objectstore.rawbluestore import RawBlueStore as RAWActivate
+from ceph_volume.objectstore.lvm import Lvm as LVMActivate
+from ceph_volume.objectstore.raw import Raw as RAWActivate
 from ceph_volume.devices.simple.activate import Activate as SimpleActivate
 
 

src/ceph-volume/ceph_volume/devices/lvm/batch.py

@@ -373,6 +373,7 @@ def main(self) -> None:
     def _execute(self, plan: List["OSD"]) -> None:
         defaults = common.get_default_args()
         global_args = [
+            'objectstore',
             'bluestore',
             'dmcrypt',
             'with_tpm',
@@ -417,8 +418,7 @@ def get_deployment_layout(self) -> List["OSD"]:
             return plan
         requested_osds = self.args.osds_per_device * len(phys_devs) + len(lvm_devs)
 
-        if self.args.objectstore == 'bluestore':
-            fast_type = 'block_db'
+        fast_type = 'block_db'
         fast_allocations = self.fast_allocations(fast_devices,
                                                  requested_osds,
                                                  num_osds,

src/ceph-volume/ceph_volume/devices/lvm/zap.py

@@ -23,7 +23,8 @@ def zap_device(path: str) -> None:
     Args:
         path (str): The path to the device to zap.
     """
-    zap_bluestore(path)
+    if disk.has_bluestore_label(path):
+        zap_bluestore(path)
     wipefs(path)
     zap_data(path)
 

src/ceph-volume/ceph_volume/objectstore/__init__.py

@@ -1,13 +1,19 @@
-from . import lvmbluestore
-from . import rawbluestore
+from . import lvm
+from . import raw
 from typing import Any, Dict
+from enum import Enum
 
 
+class ObjectStore(str, Enum):
+    bluestore: str = 'bluestore'
+    seastore: str = 'seastore'
+
 mapping: Dict[str, Any] = {
     'LVM': {
-        'bluestore': lvmbluestore.LvmBlueStore
+        ObjectStore.bluestore: lvm.Lvm,
+        ObjectStore.seastore: lvm.Lvm
     },
     'RAW': {
-        'bluestore': rawbluestore.RawBlueStore
+        ObjectStore.bluestore: raw.Raw
     }
 }

src/ceph-volume/ceph_volume/objectstore/baseobjectstore.py

@@ -32,12 +32,18 @@ def __init__(self, args: "argparse.Namespace") -> None:
         self.osd_id: str = ''
         self.osd_fsid: str = ''
         self.cephx_lockbox_secret: str = ''
-        self.objectstore: str = ''
+        self.objectstore: str = getattr(args, "objectstore", '')
         self.osd_mkfs_cmd: List[str] = []
         self.block_device_path: str = ''
         self.dmcrypt_key: str = encryption_utils.create_dmcrypt_key()
         self.with_tpm: int = int(getattr(self.args, 'with_tpm', False))
         self.method: str = ''
+        self.osd_path: str = ''
+        self.key: Optional[str] = None
+        self.block_device_path: str = ''
+        self.wal_device_path: str = ''
+        self.db_device_path: str = ''
+        self.block_lv: Optional[Volume] = None
         if getattr(self.args, 'dmcrypt', False):
             self.encrypted = 1
             if not self.with_tpm:
@@ -68,7 +74,41 @@ def safe_prepare(self, args: Optional["argparse.Namespace"] = None) -> None:
         raise NotImplementedError()
 
     def add_objectstore_opts(self) -> None:
-        raise NotImplementedError()
+        """
+        Create the files for the OSD to function. A normal call will look like:
+
+            ceph-osd --cluster ceph --mkfs --mkkey -i 0 \
+                    --monmap /var/lib/ceph/osd/ceph-0/activate.monmap \
+                    --osd-data /var/lib/ceph/osd/ceph-0 \
+                    --osd-uuid 8d208665-89ae-4733-8888-5d3bfbeeec6c \
+                    --keyring /var/lib/ceph/osd/ceph-0/keyring \
+                    --setuser ceph --setgroup ceph
+
+        In some cases it is required to use the keyring, when it is passed
+        in as a keyword argument it is used as part of the ceph-osd command
+        """
+
+        if self.wal_device_path:
+            self.osd_mkfs_cmd.extend(
+                ['--bluestore-block-wal-path', self.wal_device_path]
+            )
+            system.chown(self.wal_device_path)
+
+        if self.db_device_path:
+            self.osd_mkfs_cmd.extend(
+                ['--bluestore-block-db-path', self.db_device_path]
+            )
+            system.chown(self.db_device_path)
+
+        if self.get_osdspec_affinity():
+            self.osd_mkfs_cmd.extend(['--osdspec-affinity',
+                                      self.get_osdspec_affinity()])
+
+    def unlink_bs_symlinks(self) -> None:
+        for link_name in ['block', 'block.db', 'block.wal']:
+            link_path = os.path.join(self.osd_path, link_name)
+            if os.path.exists(link_path):
+                os.unlink(os.path.join(self.osd_path, link_name))
 
     def prepare_osd_req(self, tmpfs: bool = True) -> None:
         # create the directory
@@ -116,10 +156,8 @@ def build_osd_mkfs_cmd(self) -> List[str]:
         ]
         if self.cephx_secret is not None:
             self.osd_mkfs_cmd.extend(['--keyfile', '-'])
-        try:
-            self.add_objectstore_opts()
-        except NotImplementedError:
-            logger.info("No specific objectstore options to add.")
+
+        self.add_objectstore_opts()
 
         self.osd_mkfs_cmd.extend(self.supplementary_command)
         return self.osd_mkfs_cmd
@@ -154,6 +192,20 @@ def osd_mkfs(self) -> None:
                     raise RuntimeError('Command failed with exit code %s: %s' %
                                        (returncode, ' '.join(cmd)))
 
+        mapping: Dict[str, Any] = {'raw': ['data', 'block_db', 'block_wal'],
+                                   'lvm': ['ceph.block_device', 'ceph.db_device', 'ceph.wal_device']}
+        if self.args.dmcrypt:
+            for dev_type in mapping[self.method]:
+                if self.method == 'raw':
+                    path = self.args.__dict__.get(dev_type, None)
+                else:
+                    if self.block_lv is not None:
+                        path = self.block_lv.tags.get(dev_type, None)
+                    else:
+                        raise RuntimeError('Unexpected error while running bluestore mkfs.')
+                if path is not None:
+                    encryption_utils.CephLuks2(path).config_luks2({'subsystem': f'ceph_fsid={self.osd_fsid}'})
+
     def activate(self) -> None:
         raise NotImplementedError()
 
@@ -179,3 +231,30 @@ def enroll_tpm2(self, device: str) -> None:
                                   device, '--unlock-key-file', temp_file_name,
                                   '--tpm2-pcrs', '9+12', '--wipe-slot', 'tpm2']
                 process.call(cmd, run_on_host=True, show_command=True)
+
+    def add_label(self, key: str,
+                  value: str,
+                  device: str) -> None:
+        """Add a label to a BlueStore device.
+        Args:
+            key (str): The name of the label being added.
+            value (str): Value of the label being added.
+            device (str): The path of the BlueStore device.
+        Raises:
+            RuntimeError: If `ceph-bluestore-tool` command doesn't success.
+        """
+
+        command: List[str] = ['ceph-bluestore-tool',
+                              'set-label-key',
+                              '-k',
+                              key,
+                              '-v',
+                              value,
+                              '--dev',
+                              device]
+
+        _, err, rc = process.call(command,
+                                  terminal_verbose=True,
+                                  show_command=True)
+        if rc:
+            raise RuntimeError(f"Can't add BlueStore label '{key}' to device {device}: {err}")

src/ceph-volume/ceph_volume/objectstore/bluestore.py

@@ -9,7 +9,7 @@
 from ceph_volume.systemd import systemctl
 from ceph_volume.devices.lvm.common import rollback_osd
 from ceph_volume.devices.lvm.listing import direct_report
-from .bluestore import BlueStore
+from .baseobjectstore import BaseObjectStore
 from typing import Dict, Any, Optional, List, TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -19,7 +19,7 @@
 logger = logging.getLogger(__name__)
 
 
-class LvmBlueStore(BlueStore):
+class Lvm(BaseObjectStore):
     def __init__(self, args: "argparse.Namespace") -> None:
         super().__init__(args)
         self.method = 'lvm'
@@ -59,6 +59,7 @@ def pre_prepare(self) -> None:
             self.block_lv = self.prepare_data_device('block', self.osd_fsid)
         self.block_device_path = self.block_lv.__dict__['lv_path']
 
+        self.tags['ceph.objectstore'] = self.objectstore
         self.tags['ceph.block_device'] = self.block_lv.__dict__['lv_path']
         self.tags['ceph.block_uuid'] = self.block_lv.__dict__['lv_uuid']
         self.tags['ceph.cephx_lockbox_secret'] = self.cephx_lockbox_secret
@@ -392,12 +393,14 @@ def _activate(self,
         # ``prime-osd-dir`` can succeed even if permissions are
         # somehow messed up.
         system.chown(self.osd_path)
-        prime_command = [
-            'ceph-bluestore-tool', '--cluster=%s' % conf.cluster,
-            'prime-osd-dir', '--dev', osd_lv_path,
-            '--path', self.osd_path, '--no-mon-config']
-
-        process.run(prime_command)
+        objectstore = osd_block_lv.tags.get('ceph.objectstore', 'bluestore')
+        if objectstore == 'bluestore':
+            prime_command = [
+                'ceph-bluestore-tool', '--cluster=%s' % conf.cluster,
+                'prime-osd-dir', '--dev', osd_lv_path,
+                '--path', self.osd_path, '--no-mon-config']
+
+            process.run(prime_command)
         # always re-do the symlink regardless if it exists, so that the block,
         # block.wal, and block.db devices that may have changed can be mapped
         # correctly every time

…/ceph_volume/objectstore/lvmbluestore.py …ph-volume/ceph_volume/objectstore/lvm.pysrc/ceph-volume/ceph_volume/objectstore/lvmbluestore.py renamed to src/ceph-volume/ceph_volume/objectstore/lvm.py src/ceph-volume/ceph_volume/objectstore/lvmbluestore.py renamed to src/ceph-volume/ceph_volume/objectstore/lvm.py

@@ -1,7 +1,7 @@
 import logging
 import json
 import os
-from .bluestore import BlueStore
+from .baseobjectstore import BaseObjectStore
 from ceph_volume import terminal, decorators, conf, process
 from ceph_volume.util import system, disk
 from ceph_volume.util import prepare as prepare_utils
@@ -17,7 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
-class RawBlueStore(BlueStore):
+class Raw(BaseObjectStore):
     def __init__(self, args: "argparse.Namespace") -> None:
         super().__init__(args)
         self.method = 'raw'