Merge pull request ceph#52218 from YiteGu/cephfs-get-statfs

mgr/dashboard: CephFS statfs REST API

Reviewed-by: Avan Thakkar <[email protected]>
Reviewed-by: Dhairya Parmar <[email protected]>
Reviewed-by: Nizamudeen A <[email protected]>
Reviewed-by: Pere Diaz Bou <[email protected]>

Author: nizamial09

qa/tasks/mgr/dashboard/test_cephfs.py

@@ -74,6 +74,26 @@ def new_quota_dir(self):
         yield 1
         self.rm_dir(self.QUOTA_PATH)
 
+    def write_to_file(self, path, buf):
+        params = {'path': path, 'buf': buf}
+        self._post(f"/api/cephfs/{self.get_fs_id()}/write_to_file",
+                   params=params)
+        self.assertStatus(200)
+
+    def unlink(self, path, expectedStatus=200):
+        params = {'path': path}
+        self._delete(f"/api/cephfs/{self.get_fs_id()}/unlink",
+                     params=params)
+        self.assertStatus(expectedStatus)
+
+    def statfs(self, path):
+        params = {'path': path}
+        data = self._get(f"/api/cephfs/{self.get_fs_id()}/statfs",
+                         params=params)
+        self.assertStatus(200)
+        self.assertIsInstance(data, dict)
+        return data
+
     @DashboardTestCase.RunAs('test', 'test', ['block-manager'])
     def test_access_permissions(self):
         fs_id = self.get_fs_id()
@@ -290,3 +310,40 @@ def test_listing_of_ui_api_ls_on_deeper_levels(self):
         ui_api_ls = self.ui_ls_dir('/pictures', 0)
         self.assertEqual(api_ls, ui_api_ls)
         self.rm_dir('/pictures')
+
+    def test_statfs(self):
+        self.statfs('/')
+
+        self.mk_dirs('/animal')
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 0)
+        self.assertEqual(stats['files'], 0)
+        self.assertEqual(stats['subdirs'], 1)
+
+        buf = 'a' * 512
+        self.write_to_file('/animal/lion', buf)
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 512)
+        self.assertEqual(stats['files'], 1)
+        self.assertEqual(stats['subdirs'], 1)
+
+        buf = 'b' * 512
+        self.write_to_file('/animal/tiger', buf)
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 1024)
+        self.assertEqual(stats['files'], 2)
+        self.assertEqual(stats['subdirs'], 1)
+
+        self.unlink('/animal/tiger')
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 512)
+        self.assertEqual(stats['files'], 1)
+        self.assertEqual(stats['subdirs'], 1)
+
+        self.unlink('/animal/lion')
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 0)
+        self.assertEqual(stats['files'], 0)
+        self.assertEqual(stats['subdirs'], 1)
+
+        self.rm_dir('/animal')

src/pybind/mgr/dashboard/controllers/cephfs.py

@@ -19,6 +19,11 @@
     'max_bytes': (int, ''),
     'max_files': (int, '')
 }
+GET_STATFS_SCHEMA = {
+    'bytes': (int, ''),
+    'files': (int, ''),
+    'subdirs': (int, '')
+}
 
 logger = logging.getLogger("controllers.rgw")
 
@@ -461,6 +466,47 @@ def get_quota(self, fs_id, path):
         cfs = self._cephfs_instance(fs_id)
         return cfs.get_quotas(path)
 
+    @RESTController.Resource('POST', path='/write_to_file')
+    @allow_empty_body
+    def write_to_file(self, fs_id, path, buf) -> None:
+        """
+        Write some data to the specified path.
+        :param fs_id: The filesystem identifier.
+        :param path: The path of the file to write.
+        :param buf: The str to write to the buf.
+        """
+        cfs = self._cephfs_instance(fs_id)
+        cfs.write_to_file(path, buf)
+
+    @RESTController.Resource('DELETE', path='/unlink')
+    def unlink(self, fs_id, path) -> None:
+        """
+        Removes a file, link, or symbolic link.
+        :param fs_id: The filesystem identifier.
+        :param path: The path of the file or link to unlink.
+        """
+        cfs = self._cephfs_instance(fs_id)
+        cfs.unlink(path)
+
+    @RESTController.Resource('GET', path='/statfs')
+    @EndpointDoc("Get Cephfs statfs of the specified path",
+                 parameters={
+                     'fs_id': (str, 'File System Identifier'),
+                     'path': (str, 'File System Path'),
+                 },
+                 responses={200: GET_STATFS_SCHEMA})
+    def statfs(self, fs_id, path) -> dict:
+        """
+        Get the statfs of the specified path.
+        :param fs_id: The filesystem identifier.
+        :param path: The path of the directory/file.
+        :return: Returns a dictionary containing 'bytes',
+        'files' and 'subdirs'.
+        :rtype: dict
+        """
+        cfs = self._cephfs_instance(fs_id)
+        return cfs.statfs(path)
+
     @RESTController.Resource('POST', path='/snapshot')
     @allow_empty_body
     def snapshot(self, fs_id, path, name=None):

src/pybind/mgr/dashboard/openapi.yaml

@@ -2031,6 +2031,60 @@ paths:
       - jwt: []
       tags:
       - Cephfs
+  /api/cephfs/{fs_id}/statfs:
+    get:
+      description: "\n        Get the statfs of the specified path.\n        :param\
+        \ fs_id: The filesystem identifier.\n        :param path: The path of the\
+        \ directory/file.\n        :return: Returns a dictionary containing 'bytes',\n\
+        \        'files' and 'subdirs'.\n        :rtype: dict\n        "
+      parameters:
+      - description: File System Identifier
+        in: path
+        name: fs_id
+        required: true
+        schema:
+          type: string
+      - description: File System Path
+        in: query
+        name: path
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              schema:
+                properties:
+                  bytes:
+                    description: ''
+                    type: integer
+                  files:
+                    description: ''
+                    type: integer
+                  subdirs:
+                    description: ''
+                    type: integer
+                required:
+                - bytes
+                - files
+                - subdirs
+                type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Get Cephfs statfs of the specified path
+      tags:
+      - Cephfs
   /api/cephfs/{fs_id}/tree:
     delete:
       description: "\n        Remove a directory.\n        :param fs_id: The filesystem\
@@ -2113,6 +2167,95 @@ paths:
       - jwt: []
       tags:
       - Cephfs
+  /api/cephfs/{fs_id}/unlink:
+    delete:
+      description: "\n        Removes a file, link, or symbolic link.\n        :param\
+        \ fs_id: The filesystem identifier.\n        :param path: The path of the\
+        \ file or link to unlink.\n        "
+      parameters:
+      - in: path
+        name: fs_id
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: path
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Cephfs
+  /api/cephfs/{fs_id}/write_to_file:
+    post:
+      description: "\n        Write some data to the specified path.\n        :param\
+        \ fs_id: The filesystem identifier.\n        :param path: The path of the\
+        \ file to write.\n        :param buf: The str to write to the buf.\n     \
+        \   "
+      parameters:
+      - in: path
+        name: fs_id
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                buf:
+                  type: string
+                path:
+                  type: string
+              required:
+              - path
+              - buf
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Cephfs
   /api/cluster:
     get:
       parameters: []

src/pybind/mgr/dashboard/services/cephfs.py

@@ -3,7 +3,7 @@
 import datetime
 import logging
 import os
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 
 import cephfs
 
@@ -260,3 +260,45 @@ def set_quotas(self, path, max_bytes=None, max_files=None):
         if max_files is not None:
             self.cfs.setxattr(path, 'ceph.quota.max_files',
                               str(max_files).encode(), 0)
+
+    def write_to_file(self, path, buf) -> None:
+        """
+        Write some data to the specified path.
+        :param path: The path of the file to write.
+        :type path: str.
+        :param buf: The str to write to the buf.
+        :type buf: str.
+        """
+        try:
+            fd = self.cfs.open(path, 'w', 644)
+            self.cfs.write(fd, buf.encode('utf-8'), 0)
+        except cephfs.Error as e:
+            logger.debug("EIO: %s", str(e))
+        finally:
+            if fd is not None:
+                self.cfs.close(fd)
+
+    def unlink(self, path) -> None:
+        """
+        Removes a file, link, or symbolic link.
+        :param path: The path of the file or link to unlink.
+        """
+        self.cfs.unlink(path)
+
+    def statfs(self, path) -> dict:
+        """
+        Get the statfs of the specified path by xattr.
+        :param path: The path of the directory/file.
+        :type path: str
+        :return: Returns a dictionary containing 'bytes',
+            'files' and 'subdirs'.
+        :rtype: dict
+        """
+        rbytes = 0
+        rfiles = 0
+        rsubdirs = 0
+        with suppress(cephfs.NoData):
+            rbytes = int(self.cfs.getxattr(path, 'ceph.dir.rbytes'))
+            rfiles = int(self.cfs.getxattr(path, 'ceph.dir.rfiles'))
+            rsubdirs = int(self.cfs.getxattr(path, 'ceph.dir.rsubdirs'))
+        return {'bytes': rbytes, 'files': rfiles, 'subdirs': rsubdirs}