Merge branch 'ceph:main' into main

Author: gbregman

doc/dev/macos.rst

@@ -7,7 +7,7 @@ Since we've switched to C++ 17, and the default clang shipped with Xcode 9 does
 
 and install all the necessary bits::
 
-  brew install snappy ccache cmake pkg-config
+  brew install snappy ccache cmake pkg-config bash gnu-getopt
   pip install cython
 
 install FUSE if you want to build the FUSE support::

doc/mgr/smb.rst

@@ -585,7 +585,7 @@ source_type
     Optional. One of ``resource`` (the default) or ``empty``
 ref
     String. Required for ``source_type: resource``. Must refer to the ID of a
-    ``ceph.smb.join.auth`` resource
+    ``ceph.smb.usersgroups`` resource
 
 .. _tls-source-fields:
 

src/ceph-volume/ceph_volume/tests/util/test_disk.py

@@ -256,7 +256,11 @@ def test_no_devices_are_found(self, tmpdir, patched_get_block_devs_sysfs):
     def test_sda_block_is_found(self, patched_get_block_devs_sysfs, fake_filesystem):
         sda_path = '/dev/sda'
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert len(result.keys()) == 1
         assert result[sda_path]['human_readable_size'] == '0.00 B'
         assert result[sda_path]['model'] == ''
@@ -266,7 +270,11 @@ def test_sda_size(self, patched_get_block_devs_sysfs, fake_filesystem):
         sda_path = '/dev/sda'
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
         fake_filesystem.create_file('/sys/block/sda/size', contents = '1024')
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert list(result.keys()) == [sda_path]
         assert result[sda_path]['human_readable_size'] == '512.00 KB'
 
@@ -275,30 +283,46 @@ def test_sda_sectorsize_fallsback(self, patched_get_block_devs_sysfs, fake_files
         sda_path = '/dev/sda'
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
         fake_filesystem.create_file('/sys/block/sda/queue/hw_sector_size', contents = '1024')
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert list(result.keys()) == [sda_path]
         assert result[sda_path]['sectorsize'] == '1024'
 
     def test_sda_sectorsize_from_logical_block(self, patched_get_block_devs_sysfs, fake_filesystem):
         sda_path = '/dev/sda'
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
         fake_filesystem.create_file('/sys/block/sda/queue/logical_block_size', contents = '99')
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert result[sda_path]['sectorsize'] == '99'
 
     def test_sda_sectorsize_does_not_fallback(self, patched_get_block_devs_sysfs, fake_filesystem):
         sda_path = '/dev/sda'
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
         fake_filesystem.create_file('/sys/block/sda/queue/logical_block_size', contents = '99')
         fake_filesystem.create_file('/sys/block/sda/queue/hw_sector_size', contents = '1024')
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert result[sda_path]['sectorsize'] == '99'
 
     def test_is_rotational(self, patched_get_block_devs_sysfs, fake_filesystem):
         sda_path = '/dev/sda'
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
         fake_filesystem.create_file('/sys/block/sda/queue/rotational', contents = '1')
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert result[sda_path]['rotational'] == '1'
 
     def test_is_ceph_rbd(self, patched_get_block_devs_sysfs, fake_filesystem):
@@ -313,7 +337,11 @@ def test_actuator_device(self, patched_get_block_devs_sysfs, fake_filesystem):
         patched_get_block_devs_sysfs.return_value = [[sda_path, sda_path, 'disk', sda_path]]
         for actuator in range(0, fake_actuator_nb):
             fake_filesystem.create_dir(f'/sys/block/sda/queue/independent_access_ranges/{actuator}')
-        result = disk.get_devices()
+        with patch("ceph_volume.util.disk.UdevData") as MockUdevData:
+            mock_instance = MagicMock()
+            mock_instance.is_lvm = False
+            MockUdevData.return_value = mock_instance
+            result = disk.get_devices()
         assert result[sda_path]['actuators'] == fake_actuator_nb
 
 
@@ -668,6 +696,7 @@ def setUp(self) -> None:
         self.fs.create_file(self.fake_device, st_mode=(stat.S_IFBLK | 0o600))
         self.fs.create_file('/run/udev/data/b999:0', create_missing_dirs=True, contents=udev_data_bare_device)
         self.fs.create_file('/run/udev/data/b998:1', create_missing_dirs=True, contents=udev_data_lv_device)
+        self.fs.create_file('/run/udev/data/b997:2', create_missing_dirs=True, contents="")
 
     def test_device_not_found(self) -> None:
         self.fs.remove(self.fake_device)
@@ -682,6 +711,13 @@ def test_no_data(self) -> None:
         with pytest.raises(RuntimeError):
             disk.UdevData(self.fake_device)
 
+    @patch('ceph_volume.util.disk.os.stat', MagicMock())
+    @patch('ceph_volume.util.disk.os.minor', Mock(return_value=2))
+    @patch('ceph_volume.util.disk.os.major', Mock(return_value=997))
+    def test_empty_data(self) -> None:
+        # no exception should be raised when a /run/udev/data/* file is empty
+        _ = disk.UdevData(self.fake_device)
+
     @patch('ceph_volume.util.disk.os.stat', MagicMock())
     @patch('ceph_volume.util.disk.os.minor', Mock(return_value=0))
     @patch('ceph_volume.util.disk.os.major', Mock(return_value=999))

src/ceph-volume/ceph_volume/util/disk.py

@@ -5,7 +5,6 @@
 import time
 import json
 from ceph_volume import process, allow_loop_devices
-from ceph_volume.api import lvm
 from ceph_volume.util.system import get_file_contents
 from typing import Dict, List, Any, Union, Optional
 
@@ -802,9 +801,12 @@ def get_devices(_sys_block_path='/sys/block', device=''):
             continue
 
         # If the mapper device is a logical volume it gets excluded
-        if is_mapper_device(diskname):
-            if lvm.get_device_lvs(diskname):
+        try:
+            if UdevData(diskname).is_lvm:
                 continue
+        except RuntimeError:
+            logger.debug("get_devices(): device {} couldn't be found.".format(diskname))
+            continue
 
         # all facts that have no defaults
         # (<name>, <path relative to _sys_block_path>)
@@ -1372,7 +1374,7 @@ def __init__(self, path: str) -> None:
 
         with open(self.udev_data_path, 'r') as f:
             content: str = f.read().strip()
-            self.raw_data: List[str] = content.split('\n')
+            self.raw_data: List[str] = content.split('\n') if content else []
 
         for line in self.raw_data:
             data_type, data = line.split(':', 1)

src/osd/OSDMap.cc

@@ -374,7 +374,7 @@ bool OSDMap::subtree_type_is_down(
 {
   if (id >= 0) {
     bool is_down_ret = is_down(id);
-    if (!is_out(id)) {
+    if (!is_out(id) && !(osd_state[id] & CEPH_OSD_NEW)) {
       if (is_down_ret) {
         down_in_osds->insert(id);
       } else {

src/rgw/driver/rados/rgw_dedup.cc

@@ -344,7 +344,7 @@ namespace rgw::dedup {
       return ret;
     }
 
-    ret = ioctx.application_enable("rgw_dedup", false);
+    ret = ioctx.application_enable("rgw", false);
     if (ret == 0) {
       ldpp_dout(dpp, 10) << __func__ << "::pool " << dedup_pool.name
                          << " was associated with dedup app" << dendl;
@@ -1499,23 +1499,24 @@ namespace rgw::dedup {
                                                    const rgw_bucket_dir_entry &entry,
                                                    worker_stats_t             *p_worker_stats /*IN-OUT*/)
   {
-    // ceph store full blocks so need to round up and multiply by block_size
-    uint64_t ondisk_byte_size = calc_on_disk_byte_size(entry.meta.size);
-    // count all objects including too small and non default storage_class objs
-    p_worker_stats->ingress_obj++;
-    p_worker_stats->ingress_obj_bytes += ondisk_byte_size;
-
     parsed_etag_t parsed_etag;
     if (unlikely(!parse_etag_string(entry.meta.etag, &parsed_etag))) {
       p_worker_stats->ingress_corrupted_etag++;
-      ldpp_dout(dpp, 1) << __func__ << "::ERROR: corrupted etag" << dendl;
+      ldpp_dout(dpp, 1) << __func__ << "::ERROR: corrupted etag:" << entry.meta.etag
+                        << "::" << p_bucket->get_name() << "/" << entry.key.name << dendl;
       return -EINVAL;
     }
 
     if (unlikely((cct->_conf->subsys.should_gather<ceph_subsys_rgw_dedup, 20>()))) {
       show_ingress_bucket_idx_obj(dpp, parsed_etag, p_bucket->get_name(), entry.key.name);
     }
 
+    // ceph store full blocks so need to round up and multiply by block_size
+    uint64_t ondisk_byte_size = calc_on_disk_byte_size(entry.meta.size);
+    // count all objects including too small and non default storage_class objs
+    p_worker_stats->ingress_obj++;
+    p_worker_stats->ingress_obj_bytes += ondisk_byte_size;
+
     // We limit dedup to objects from the same storage_class
     // TBD:
     // Should we use a skip-list of storage_classes we should skip (like glacier) ?

src/rgw/driver/rados/rgw_dedup_utils.cc

@@ -310,8 +310,14 @@ namespace rgw::dedup {
                          this->non_default_storage_class_objs_bytes);
       }
       else {
-        ceph_assert(this->default_storage_class_objs == this->ingress_obj);
-        ceph_assert(this->default_storage_class_objs_bytes == this->ingress_obj_bytes);
+        if (this->default_storage_class_objs != this->ingress_obj) {
+          f->dump_unsigned("default storage class objs",
+                           this->default_storage_class_objs);
+        }
+        if (this->default_storage_class_objs_bytes != this->ingress_obj_bytes) {
+          f->dump_unsigned("default storage class objs bytes",
+                           this->default_storage_class_objs_bytes);
+        }
       }
     }