Merge pull request ceph#62105 from xhernandez/configure-case-sensitivity

pybind/mgr/volumes: configure case sensitivity

Reviewed-by: Avan Thakkar <athakkar@redhat.
Reviewed-by: Patrick Donnelly <[email protected]>
Reviewed-by: Anoop C S <[email protected]>

Author: avanthakkar

doc/cephfs/fs-volumes.rst

@@ -287,7 +287,7 @@ Use a command of the following form to create a subvolume:
 
 .. prompt:: bash #
 
-   ceph fs subvolume create <vol_name> <subvol_name> [--size <size_in_bytes>] [--group_name <subvol_group_name>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>] [--namespace-isolated] [--earmark <earmark>]
+   ceph fs subvolume create <vol_name> <subvol_name> [--size <size_in_bytes>] [--group_name <subvol_group_name>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>] [--namespace-isolated] [--earmark <earmark>] [--normalization <form>] [--case-insensitive]
 
 
 The command succeeds even if the subvolume already exists.
@@ -327,6 +327,29 @@ Valid Earmarks
    be aware that user permissions and ACLs associated with the previous scope might still apply. Ensure that
    any necessary permissions are updated as needed to maintain proper access control.
 
+When creating a subvolume you can also specify an unicode normalization form by
+using the ``--normalization`` option. This will be used to internally mangle
+file names so that unicode characters that can be represented by different
+unicode code point sequences are all mapped to the representation, which means
+that they will all access the same file. However, users will continue to see
+the same name that they used when the file was created.
+
+The valid values for the unicode normalization form are:
+
+    - nfd: canonical decomposition (default)
+    - nfc: canonical decomposition, followed by canonical composition
+    - nfkd: compatibility decomposition
+    - nfkc: compatibility decomposition, followed by canonical composition
+
+To learn more about unicode normalization forms see https://unicode.org/reports/tr15
+
+It's also possible to configure a subvolume for case insensitive access when
+the ``--case-insensitive`` option is used. When this option is added, file
+names that only differ in the case of its characters will be mapped to the same
+file. The case of the file name used when the file was created is preserved.
+
+.. note:: Setting ``--case-insensitive`` option implicitly enables
+   unicode normalization on the subvolume.
 
 Removing a subvolume
 ~~~~~~~~~~~~~~~~~~~~

qa/tasks/cephfs/test_volumes.py

@@ -2686,6 +2686,70 @@ def test_earmark_on_deleted_subvolume_with_retained_snapshot(self):
             except CommandFailedError as ce:
                 self.assertEqual(ce.exitstatus, errno.ENOENT, error_message)
 
+    def test_subvolume_create_without_normalization(self):
+        # create subvolume
+        subvolume = self._gen_subvol_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume)
+
+        # make sure it exists
+        subvolpath = self._get_subvolume_path(self.volname, subvolume)
+        self.assertNotEqual(subvolpath, None)
+
+        # check normalization
+        try:
+            self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "normalization")
+        except CommandFailedError as ce:
+            self.assertEqual(ce.exitstatus, errno.ENODATA)
+        else:
+            self.fail("expected the 'fs subvolume charmap' command to fail")
+
+    def test_subvolume_create_with_normalization(self):
+        # create subvolume
+        subvolume = self._gen_subvol_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume, "--normalization", "nfc")
+
+        # make sure it exists
+        subvolpath = self._get_subvolume_path(self.volname, subvolume)
+        self.assertNotEqual(subvolpath, None)
+
+        # check normalization
+        normalization = self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "normalization")
+        self.assertEqual(normalization.strip(), "nfc")
+
+    def test_subvolume_create_without_case_sensitivity(self):
+        # create subvolume
+        subvolume = self._gen_subvol_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume)
+
+        # make sure it exists
+        subvolpath = self._get_subvolume_path(self.volname, subvolume)
+        self.assertNotEqual(subvolpath, None)
+
+        # check case sensitivity
+        try:
+            self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "casesensitive")
+        except CommandFailedError as ce:
+            self.assertEqual(ce.exitstatus, errno.ENODATA)
+        else:
+            self.fail("expected the 'fs subvolume charmap' command to fail")
+
+    def test_subvolume_create_with_case_insensitive(self):
+        # create subvolume
+        subvolume = self._gen_subvol_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume, "--case-insensitive")
+
+        # make sure it exists
+        subvolpath = self._get_subvolume_path(self.volname, subvolume)
+        self.assertNotEqual(subvolpath, None)
+
+        # check case sensitivity
+        case_sensitive = self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "casesensitive")
+        self.assertEqual(case_sensitive.strip(), "0")
+
+        # check normalization (it's implicitly enabled by --case-insensitive, with default value 'nfd')
+        normalization = self._fs_cmd("subvolume", "charmap", "get", self.volname, subvolume, "normalization")
+        self.assertEqual(normalization.strip(), "nfd")
+
     def test_subvolume_expand(self):
         """
         That a subvolume can be expanded in size and its quota matches the expected size.

src/pybind/mgr/volumes/fs/operations/subvolume.py

@@ -5,7 +5,7 @@
 from .template import SubvolumeOpType
 from .versions import loaded_subvolumes
 
-def create_subvol(mgr, fs, vol_spec, group, subvolname, size, isolate_nspace, pool, mode, uid, gid, earmark):
+def create_subvol(mgr, fs, vol_spec, group, subvolname, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, case_insensitive):
     """
     create a subvolume (create a subvolume with the max known version).
 
@@ -19,10 +19,12 @@ def create_subvol(mgr, fs, vol_spec, group, subvolname, size, isolate_nspace, po
     :param uid: the user identifier
     :param gid: the group identifier
     :param earmark: metadata string to identify if subvolume is associated with nfs/smb
+    :param normalization: the unicode normalization form to use (nfd, nfc, nfkd or nfkc)
+    :param case_insensitive: whether to make the subvolume case insensitive or not
     :return: None
     """
     subvolume = loaded_subvolumes.get_subvolume_object_max(mgr, fs, vol_spec, group, subvolname)
-    subvolume.create(size, isolate_nspace, pool, mode, uid, gid, earmark)
+    subvolume.create(size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, case_insensitive)
 
 
 def create_clone(mgr, fs, vol_spec, group, subvolname, pool, source_volume, source_subvolume, snapname):

src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py

@@ -203,6 +203,19 @@ def get_attrs(self, pathname):
         except EarmarkException:
             attrs["earmark"] = ''
 
+        try:
+            attrs["normalization"] = self.fs.getxattr(pathname,
+                                                      'ceph.dir.normalization'
+                                                      ).decode('utf-8')
+        except cephfs.NoData:
+            attrs["normalization"] = None
+
+        try:
+            case_insensitive = self.fs.getxattr(pathname, 'ceph.dir.caseinsensitive').decode('utf-8')
+            attrs["case_insensitive"] = case_insensitive == "0"
+        except cephfs.NoData:
+            attrs["case_insensitive"] = False
+
         return attrs
 
     def set_attrs(self, path, attrs):
@@ -294,6 +307,20 @@ def set_attrs(self, path, attrs):
             fs_earmark = CephFSVolumeEarmarking(self.fs, path)
             fs_earmark.set_earmark(earmark)
 
+        normalization = attrs.get("normalization")
+        if normalization is not None:
+            try:
+                self.fs.setxattr(path, "ceph.dir.normalization", normalization.encode('utf-8'), 0)
+            except cephfs.Error as e:
+                raise VolumeException(-e.args[0], e.args[1])
+
+        case_insensitive = attrs.get("case_insensitive")
+        if case_insensitive:
+            try:
+                self.fs.setxattr(path, "ceph.dir.casesensitive", "0".encode('utf-8'), 0)
+            except cephfs.Error as e:
+                raise VolumeException(-e.args[0], e.args[1])
+
     def _resize(self, path, newsize, noshrink):
         try:
             newsize = int(newsize)
@@ -452,6 +479,21 @@ def info(self):
         except EarmarkException:
             earmark = ''
 
+        try:
+            normalization = self.fs.getxattr(subvolpath,
+                                             'ceph.dir.normalization'
+                                             ).decode('utf-8')
+        except cephfs.NoData:
+            normalization = "none"
+
+        try:
+            case_insensitive = self.fs.getxattr(subvolpath,
+                                                'ceph.dir.casesensitive'
+                                                ).decode('utf-8')
+            case_insensitive = case_insensitive == "0"
+        except cephfs.NoData:
+            case_insensitive = False
+
         return {'path': subvolpath,
                 'type': etype.value,
                 'uid': int(st["uid"]),
@@ -470,7 +512,10 @@ def info(self):
                 'pool_namespace': pool_namespace,
                 'features': self.features,
                 'state': self.state.value,
-                'earmark': earmark}
+                'earmark': earmark,
+                'normalization': normalization,
+                'case_insensitive': case_insensitive,
+        }
 
     def set_user_metadata(self, keyname, value):
         try:

src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py

@@ -85,7 +85,7 @@ def snapshot_data_path(self, snapname):
         """ Path to user data directory within a subvolume snapshot named 'snapname' """
         return self.snapshot_path(snapname)
 
-    def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark):
+    def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, case_insensitive):
         subvolume_type = SubvolumeTypes.TYPE_NORMAL
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
@@ -104,7 +104,9 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark):
                 'data_pool': pool,
                 'pool_namespace': self.namespace if isolate_nspace else None,
                 'quota': size,
-                'earmark': earmark
+                'earmark': earmark,
+                'normalization': normalization,
+                'case_insensitive': case_insensitive,
             }
             self.set_attrs(subvol_path, attrs)
 

src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py

@@ -154,7 +154,7 @@ def _set_incarnation_metadata(self, subvolume_type, qpath, initial_state):
         self.metadata_mgr.update_global_section(MetadataManager.GLOBAL_META_KEY_PATH, qpath)
         self.metadata_mgr.update_global_section(MetadataManager.GLOBAL_META_KEY_STATE, initial_state.value)
 
-    def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark):
+    def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, case_insensitive):
         subvolume_type = SubvolumeTypes.TYPE_NORMAL
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
@@ -176,7 +176,9 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark):
                 'data_pool': pool,
                 'pool_namespace': self.namespace if isolate_nspace else None,
                 'quota': size,
-                'earmark': earmark
+                'earmark': earmark,
+                'normalization': normalization,
+                'case_insensitive': case_insensitive,
             }
             self.set_attrs(subvol_path, attrs)
 

src/pybind/mgr/volumes/fs/volume.py

@@ -232,12 +232,14 @@ def _create_subvolume(self, fs_handle, volname, group, subvolname, **kwargs):
         mode       = kwargs['mode']
         isolate_nspace = kwargs['namespace_isolated']
         earmark    = kwargs['earmark'] or ''  # if not set, default to empty string --> no earmark
+        normalization = kwargs['normalization']
+        case_insensitive = kwargs['case_insensitive']
 
         oct_mode = octal_str_to_decimal_int(mode)
 
         try:
             create_subvol(
-                self.mgr, fs_handle, self.volspec, group, subvolname, size, isolate_nspace, pool, oct_mode, uid, gid, earmark)
+                self.mgr, fs_handle, self.volspec, group, subvolname, size, isolate_nspace, pool, oct_mode, uid, gid, earmark, normalization, case_insensitive)
         except VolumeException as ve:
             # kick the purge threads for async removal -- note that this
             # assumes that the subvolume is moved to trashcan for cleanup on error.
@@ -256,6 +258,8 @@ def create_subvolume(self, **kwargs):
         mode       = kwargs['mode']
         isolate_nspace = kwargs['namespace_isolated']
         earmark    = kwargs['earmark'] or ''  # if not set, default to empty string --> no earmark
+        normalization = kwargs['normalization']
+        case_insensitive = kwargs['case_insensitive']
 
         try:
             with open_volume(self, volname) as fs_handle:
@@ -270,7 +274,9 @@ def create_subvolume(self, **kwargs):
                                 'data_pool': pool,
                                 'pool_namespace': subvolume.namespace if isolate_nspace else None,
                                 'quota': size,
-                                'earmark': earmark
+                                'earmark': earmark,
+                                'normalization': normalization,
+                                'case_insensitive': case_insensitive,
                             }
                             subvolume.set_attrs(subvolume.path, attrs)
                     except VolumeException as ve:

src/pybind/mgr/volumes/module.py

@@ -144,7 +144,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                    'name=gid,type=CephInt,req=false '
                    'name=mode,type=CephString,req=false '
                    'name=namespace_isolated,type=CephBool,req=false '
-                   'name=earmark,type=CephString,req=false ',
+                   'name=earmark,type=CephString,req=false '
+                   'name=normalization,type=CephChoices,strings=nfd|nfc|nfkd|nfkc,req=false '
+                   'name=case_insensitive,type=CephBool,req=false ',
             'desc': "Create a CephFS subvolume in a volume, and optionally, "
                     "with a specific size (in bytes), a specific data pool layout, "
                     "a specific mode, in a specific subvolume group and in separate "
@@ -759,7 +761,9 @@ def _cmd_fs_subvolume_create(self, inbuf, cmd):
                                         gid=cmd.get('gid', None),
                                         mode=cmd.get('mode', '755'),
                                         namespace_isolated=cmd.get('namespace_isolated', False),
-                                        earmark=cmd.get('earmark', None))
+                                        earmark=cmd.get('earmark', None),
+                                        normalization=cmd.get('normalization', None),
+                                        case_insensitive=cmd.get('case_insensitive', False))
 
     @mgr_cmd_wrap
     def _cmd_fs_subvolume_rm(self, inbuf, cmd):