Merge PR ceph#64324 into main

* refs/pull/64324/head:
	qa/cephfs: fix test_subvolume_group_charmap_inheritance test
	doc: add name mangling documentation for subvolume group creation
	qa: add tests for name mangling in subvolume group creation
	pybind/mgr: add name mangling options to subvolume group creation

Reviewed-by: Anthony D Atri <[email protected]>
Reviewed-by: Patrick Donnelly <[email protected]>
Reviewed-by: Venky Shankar <[email protected]>

Author: vshankar

doc/cephfs/fs-volumes.rst

@@ -167,7 +167,7 @@ Create a subvolume group by running the following command:
 
 .. prompt:: bash #
 
-   ceph fs subvolumegroup create <vol_name> <group_name> [--size <size_in_bytes>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]
+   ceph fs subvolumegroup create <vol_name> <group_name> [--size <size_in_bytes>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>] [--normalization <form>] [--casesensitive <bool>]
 
 The command succeeds even if the subvolume group already exists.
 
@@ -178,6 +178,30 @@ a quota on it (see :doc:`/cephfs/quota`). By default, the subvolume group
 is created with octal file mode ``755``, uid ``0``, gid ``0`` and the data pool
 layout of its parent directory.
 
+You can also specify an unicode normalization form using the ``--normalization``
+option. This will be used to internally mangle file names so that unicode
+characters that can be represented by different unicode code point sequences
+are all mapped to the same representation, which means that they will all
+access the same file. However, users will continue to see the same name that
+they used when the file was created.
+
+The valid values for the unicode normalization form are:
+
+    - nfd: canonical decomposition (default)
+    - nfc: canonical decomposition, followed by canonical composition
+    - nfkd: compatibility decomposition
+    - nfkc: compatibility decomposition, followed by canonical composition
+
+To learn more about unicode normalization forms see https://unicode.org/reports/tr15
+
+It's also possible to configure a subvolume group for case insensitive access
+when the ``--casesensitive=0`` option is used. When this option is added, file
+names that only differ in the case of its characters will be mapped to the same
+file. The case of the file name used when the file was created is preserved.
+
+.. note:: Setting ``--casesensitive=0`` option implicitly enables
+   unicode normalization on the subvolume group.
+
 Remove a subvolume group by running a command of the following form:
 
 .. prompt:: bash #

qa/tasks/cephfs/test_volumes.py

@@ -1217,6 +1217,95 @@ def test_subvolume_group_create_with_size(self):
         # remove group
         self._fs_cmd("subvolumegroup", "rm", self.volname, group)
 
+    def test_subvolume_group_create_without_normalization(self):
+        # create group
+        group = self._gen_subvol_grp_name()
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+
+        # make sure it exists
+        grouppath = self._get_subvolume_group_path(self.volname, group)
+        self.assertNotEqual(grouppath, None)
+
+        # check normalization
+        try:
+            self._fs_cmd("subvolumegroup", "charmap", "get", self.volname, group, "normalization")
+        except CommandFailedError as ce:
+            self.assertEqual(ce.exitstatus, errno.ENODATA)
+        else:
+            self.fail("expected the 'fs subvolumegroup charmap' command to fail")
+
+    def test_subvolume_group_create_with_normalization(self):
+        # create group
+        group = self._gen_subvol_grp_name()
+        self._fs_cmd("subvolumegroup", "create", self.volname, group, "--normalization", "nfc")
+
+        # make sure it exists
+        grouppath = self._get_subvolume_group_path(self.volname, group)
+        self.assertNotEqual(grouppath, None)
+
+        # check normalization
+        normalization = self._fs_cmd("subvolumegroup", "charmap", "get", self.volname, group, "normalization")
+        self.assertEqual(normalization.strip(), "nfc")
+
+    def test_subvolume_group_create_without_case_sensitivity(self):
+        # create group
+        group = self._gen_subvol_grp_name()
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+
+        # make sure it exists
+        grouppath = self._get_subvolume_group_path(self.volname, group)
+        self.assertNotEqual(grouppath, None)
+
+        # check case sensitivity
+        try:
+            self._fs_cmd("subvolumegroup", "charmap", "get", self.volname, group, "casesensitive")
+        except CommandFailedError as ce:
+            self.assertEqual(ce.exitstatus, errno.ENODATA)
+        else:
+            self.fail("expected the 'fs subvolumegroup charmap' command to fail")
+
+    def test_subvolume_group_create_with_case_insensitive(self):
+        # create group
+        group = self._gen_subvol_grp_name()
+        self._fs_cmd("subvolumegroup", "create", self.volname, group, "--casesensitive=0")
+
+        # make sure it exists
+        grouppath = self._get_subvolume_group_path(self.volname, group)
+        self.assertNotEqual(grouppath, None)
+
+        # check case sensitivity
+        case_sensitive = self._fs_cmd("subvolumegroup", "charmap", "get", self.volname, group, "casesensitive")
+        self.assertEqual(case_sensitive.strip(), "0")
+
+        # check normalization (it's implicitly enabled by --case-insensitive, with default value 'nfd')
+        normalization = self._fs_cmd("subvolumegroup", "charmap", "get", self.volname, group, "normalization")
+        self.assertEqual(normalization.strip(), "nfd")
+
+    def test_subvolume_group_charmap_inheritance(self):
+        # create group
+        group = self._gen_subvol_grp_name()
+        self._fs_cmd("subvolumegroup", "create", self.volname, group, "--casesensitive=0", "--normalization=nfc")
+
+        # make sure it exists
+        grouppath = self._get_subvolume_group_path(self.volname, group)
+        self.assertNotEqual(grouppath, None)
+
+        # create subvolume
+        subvolume = self._gen_subvol_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group)
+
+        # make sure it exists
+        subvolpath = self._get_subvolume_path(self.volname, subvolume, group)
+        self.assertNotEqual(subvolpath, None)
+
+        # check case sensitivity
+        case_sensitive = self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "casesensitive", group)
+        self.assertEqual(case_sensitive.strip(), "0")
+
+        # check normalization (it's implicitly enabled by --case-insensitive, with default value 'nfd')
+        normalization = self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "normalization", group)
+        self.assertEqual(normalization.strip(), "nfc")
+
     def test_subvolume_group_info(self):
         # tests the 'fs subvolumegroup info' command
 

src/pybind/mgr/volumes/fs/operations/group.py

@@ -126,6 +126,17 @@ def info(self):
         except cephfs.Error as e:
             raise VolumeException(-e.args[0], e.args[1])
 
+        try:
+            normalization = self.fs.getxattr(self.path, 'ceph.dir.normalization').decode('utf-8')
+        except cephfs.NoData:
+            normalization = "none"
+
+        try:
+            casesensitive = self.fs.getxattr(self.path, 'ceph.dir.casesensitive').decode('utf-8')
+            casesensitive = casesensitive == "1"
+        except cephfs.NoData:
+            casesensitive = True
+
         return {'uid': int(st["uid"]),
                 'gid': int(st["gid"]),
                 'atime': str(st["atime"]),
@@ -136,7 +147,10 @@ def info(self):
                 'created_at': str(st["btime"]),
                 'bytes_quota': "infinite" if nsize == 0 else nsize,
                 'bytes_used': int(usedbytes),
-                'bytes_pcent': "undefined" if nsize == 0 else '{0:.2f}'.format((float(usedbytes) / nsize) * 100.0)}
+                'bytes_pcent': "undefined" if nsize == 0 else '{0:.2f}'.format((float(usedbytes) / nsize) * 100.0),
+                'normalization': normalization,
+                'casesensitive': casesensitive,
+        }
 
     def resize(self, newsize, noshrink):
         try:
@@ -223,7 +237,21 @@ def set_group_attrs(fs, path, attrs):
     if mode is not None:
         fs.lchmod(path, mode)
 
-def create_group(fs, vol_spec, groupname, size, pool, mode, uid, gid):
+    normalization = attrs.get("normalization")
+    if normalization is not None:
+        try:
+            fs.setxattr(path, "ceph.dir.normalization", normalization.encode('utf-8'), 0)
+        except cephfs.Error as e:
+            raise VolumeException(-e.args[0], e.args[1])
+
+    casesensitive = attrs.get("casesensitive")
+    if casesensitive is False:
+        try:
+            fs.setxattr(path, "ceph.dir.casesensitive", "0".encode('utf-8'), 0)
+        except cephfs.Error as e:
+            raise VolumeException(-e.args[0], e.args[1])
+
+def create_group(fs, vol_spec, groupname, size, pool, mode, uid, gid, normalization, casesensitive):
     """
     create a subvolume group.
 
@@ -235,6 +263,8 @@ def create_group(fs, vol_spec, groupname, size, pool, mode, uid, gid):
     :param mode: the user permissions
     :param uid: the user identifier
     :param gid: the group identifier
+    :param normalization: the unicode normalization form to use (nfd, nfc, nfkd or nfkc)
+    :param casesensitive: whether to make the subvolume case insensitive or not
     :return: None
     """
     group = Group(fs, vol_spec, groupname)
@@ -249,7 +279,9 @@ def create_group(fs, vol_spec, groupname, size, pool, mode, uid, gid):
             'uid': uid,
             'gid': gid,
             'data_pool': pool,
-            'quota': size
+            'quota': size,
+            'normalization': normalization,
+            'casesensitive': casesensitive,
         }
         set_group_attrs(fs, path, attrs)
     except (cephfs.Error, VolumeException) as e:

src/pybind/mgr/volumes/fs/volume.py

@@ -1087,6 +1087,8 @@ def create_subvolume_group(self, **kwargs):
         uid       = kwargs['uid']
         gid       = kwargs['gid']
         mode      = kwargs['mode']
+        normalization = kwargs['normalization']
+        casesensitive = kwargs['casesensitive']
 
         try:
             with open_volume(self, volname) as fs_handle:
@@ -1098,13 +1100,15 @@ def create_subvolume_group(self, **kwargs):
                             'gid': gid,
                             'mode': octal_str_to_decimal_int(mode),
                             'data_pool': pool,
-                            'quota': size
+                            'quota': size,
+                            'normalization': normalization,
+                            'casesensitive': casesensitive,
                         }
                         set_group_attrs(fs_handle, group.path, attrs)
                 except VolumeException as ve:
                     if ve.errno == -errno.ENOENT:
                         oct_mode = octal_str_to_decimal_int(mode)
-                        create_group(fs_handle, self.volspec, groupname, size, pool, oct_mode, uid, gid)
+                        create_group(fs_handle, self.volspec, groupname, size, pool, oct_mode, uid, gid, normalization, casesensitive)
                     else:
                         raise
         except VolumeException as ve:

src/pybind/mgr/volumes/module.py

@@ -91,7 +91,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                    'name=pool_layout,type=CephString,req=false '
                    'name=uid,type=CephInt,req=false '
                    'name=gid,type=CephInt,req=false '
-                   'name=mode,type=CephString,req=false ',
+                   'name=mode,type=CephString,req=false '
+                   'name=normalization,type=CephChoices,strings=nfd|nfc|nfkd|nfkc,req=false '
+                   'name=casesensitive,type=CephBool,req=false ',
             'desc': "Create a CephFS subvolume group in a volume, and optionally, "
                     "with a specific data pool layout, and a specific numeric mode",
             'perm': 'rw'
@@ -727,7 +729,9 @@ def _cmd_fs_subvolumegroup_create(self, inbuf, cmd):
         return self.vc.create_subvolume_group(
             vol_name=cmd['vol_name'], group_name=cmd['group_name'], size=cmd.get('size', None),
             pool_layout=cmd.get('pool_layout', None), mode=cmd.get('mode', '755'),
-            uid=cmd.get('uid', None), gid=cmd.get('gid', None))
+            uid=cmd.get('uid', None), gid=cmd.get('gid', None),
+            normalization=cmd.get('normalization', None),
+            casesensitive=cmd.get('casesensitive', None))
 
     @mgr_cmd_wrap
     def _cmd_fs_subvolumegroup_rm(self, inbuf, cmd):