mgr/dashboard: CephFS statfs REST API

Introduce statfs for the CephFS REST API controller, we
can easily get statfs of the specified path by it, it returns
the used bytes, used files and used subdirs.

Fixes: https://tracker.ceph.com/issues/61883
Signed-off-by: Yite Gu <[email protected]>

Author: YiteGu

qa/tasks/mgr/dashboard/test_cephfs.py

@@ -74,6 +74,26 @@ def new_quota_dir(self):
         yield 1
         self.rm_dir(self.QUOTA_PATH)
 
+    def write_to_file(self, path, buf):
+        params = {'path': path, 'buf': buf}
+        self._post(f"/api/cephfs/{self.get_fs_id()}/write_to_file",
+                   params=params)
+        self.assertStatus(200)
+
+    def unlink(self, path, expectedStatus=200):
+        params = {'path': path}
+        self._delete(f"/api/cephfs/{self.get_fs_id()}/unlink",
+                     params=params)
+        self.assertStatus(expectedStatus)
+
+    def statfs(self, path):
+        params = {'path': path}
+        data = self._get(f"/api/cephfs/{self.get_fs_id()}/statfs",
+                         params=params)
+        self.assertStatus(200)
+        self.assertIsInstance(data, dict)
+        return data
+
     @DashboardTestCase.RunAs('test', 'test', ['block-manager'])
     def test_access_permissions(self):
         fs_id = self.get_fs_id()
@@ -290,3 +310,40 @@ def test_listing_of_ui_api_ls_on_deeper_levels(self):
         ui_api_ls = self.ui_ls_dir('/pictures', 0)
         self.assertEqual(api_ls, ui_api_ls)
         self.rm_dir('/pictures')
+
+    def test_statfs(self):
+        self.statfs('/')
+
+        self.mk_dirs('/animal')
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 0)
+        self.assertEqual(stats['files'], 0)
+        self.assertEqual(stats['subdirs'], 1)
+
+        buf = 'a' * 512
+        self.write_to_file('/animal/lion', buf)
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 512)
+        self.assertEqual(stats['files'], 1)
+        self.assertEqual(stats['subdirs'], 1)
+
+        buf = 'b' * 512
+        self.write_to_file('/animal/tiger', buf)
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 1024)
+        self.assertEqual(stats['files'], 2)
+        self.assertEqual(stats['subdirs'], 1)
+
+        self.unlink('/animal/tiger')
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 512)
+        self.assertEqual(stats['files'], 1)
+        self.assertEqual(stats['subdirs'], 1)
+
+        self.unlink('/animal/lion')
+        stats = self.statfs('/animal')
+        self.assertEqual(stats['bytes'], 0)
+        self.assertEqual(stats['files'], 0)
+        self.assertEqual(stats['subdirs'], 1)
+
+        self.rm_dir('/animal')

src/pybind/mgr/dashboard/controllers/cephfs.py

@@ -19,6 +19,11 @@
     'max_bytes': (int, ''),
     'max_files': (int, '')
 }
+GET_STATFS_SCHEMA = {
+    'bytes': (int, ''),
+    'files': (int, ''),
+    'subdirs': (int, '')
+}
 
 logger = logging.getLogger("controllers.rgw")
 
@@ -461,6 +466,47 @@ def get_quota(self, fs_id, path):
         cfs = self._cephfs_instance(fs_id)
         return cfs.get_quotas(path)
 
+    @RESTController.Resource('POST', path='/write_to_file')
+    @allow_empty_body
+    def write_to_file(self, fs_id, path, buf) -> None:
+        """
+        Write some data to the specified path.
+        :param fs_id: The filesystem identifier.
+        :param path: The path of the file to write.
+        :param buf: The str to write to the buf.
+        """
+        cfs = self._cephfs_instance(fs_id)
+        cfs.write_to_file(path, buf)
+
+    @RESTController.Resource('DELETE', path='/unlink')
+    def unlink(self, fs_id, path) -> None:
+        """
+        Removes a file, link, or symbolic link.
+        :param fs_id: The filesystem identifier.
+        :param path: The path of the file or link to unlink.
+        """
+        cfs = self._cephfs_instance(fs_id)
+        cfs.unlink(path)
+
+    @RESTController.Resource('GET', path='/statfs')
+    @EndpointDoc("Get Cephfs statfs of the specified path",
+                 parameters={
+                     'fs_id': (str, 'File System Identifier'),
+                     'path': (str, 'File System Path'),
+                 },
+                 responses={200: GET_STATFS_SCHEMA})
+    def statfs(self, fs_id, path) -> dict:
+        """
+        Get the statfs of the specified path.
+        :param fs_id: The filesystem identifier.
+        :param path: The path of the directory/file.
+        :return: Returns a dictionary containing 'bytes',
+        'files' and 'subdirs'.
+        :rtype: dict
+        """
+        cfs = self._cephfs_instance(fs_id)
+        return cfs.statfs(path)
+
     @RESTController.Resource('POST', path='/snapshot')
     @allow_empty_body
     def snapshot(self, fs_id, path, name=None):

src/pybind/mgr/dashboard/openapi.yaml

@@ -2031,6 +2031,60 @@ paths:
       - jwt: []
       tags:
       - Cephfs
+  /api/cephfs/{fs_id}/statfs:
+    get:
+      description: "\n        Get the statfs of the specified path.\n        :param\
+        \ fs_id: The filesystem identifier.\n        :param path: The path of the\
+        \ directory/file.\n        :return: Returns a dictionary containing 'bytes',\n\
+        \        'files' and 'subdirs'.\n        :rtype: dict\n        "
+      parameters:
+      - description: File System Identifier
+        in: path
+        name: fs_id
+        required: true
+        schema:
+          type: string
+      - description: File System Path
+        in: query
+        name: path
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              schema:
+                properties:
+                  bytes:
+                    description: ''
+                    type: integer
+                  files:
+                    description: ''
+                    type: integer
+                  subdirs:
+                    description: ''
+                    type: integer
+                required:
+                - bytes
+                - files
+                - subdirs
+                type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Get Cephfs statfs of the specified path
+      tags:
+      - Cephfs
   /api/cephfs/{fs_id}/tree:
     delete:
       description: "\n        Remove a directory.\n        :param fs_id: The filesystem\
@@ -2113,6 +2167,95 @@ paths:
       - jwt: []
       tags:
       - Cephfs
+  /api/cephfs/{fs_id}/unlink:
+    delete:
+      description: "\n        Removes a file, link, or symbolic link.\n        :param\
+        \ fs_id: The filesystem identifier.\n        :param path: The path of the\
+        \ file or link to unlink.\n        "
+      parameters:
+      - in: path
+        name: fs_id
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: path
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Cephfs
+  /api/cephfs/{fs_id}/write_to_file:
+    post:
+      description: "\n        Write some data to the specified path.\n        :param\
+        \ fs_id: The filesystem identifier.\n        :param path: The path of the\
+        \ file to write.\n        :param buf: The str to write to the buf.\n     \
+        \   "
+      parameters:
+      - in: path
+        name: fs_id
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                buf:
+                  type: string
+                path:
+                  type: string
+              required:
+              - path
+              - buf
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Cephfs
   /api/cluster:
     get:
       parameters: []

src/pybind/mgr/dashboard/services/cephfs.py

@@ -3,7 +3,7 @@
 import datetime
 import logging
 import os
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 
 import cephfs
 
@@ -260,3 +260,45 @@ def set_quotas(self, path, max_bytes=None, max_files=None):
         if max_files is not None:
             self.cfs.setxattr(path, 'ceph.quota.max_files',
                               str(max_files).encode(), 0)
+
+    def write_to_file(self, path, buf) -> None:
+        """
+        Write some data to the specified path.
+        :param path: The path of the file to write.
+        :type path: str.
+        :param buf: The str to write to the buf.
+        :type buf: str.
+        """
+        try:
+            fd = self.cfs.open(path, 'w', 644)
+            self.cfs.write(fd, buf.encode('utf-8'), 0)
+        except cephfs.Error as e:
+            logger.debug("EIO: %s", str(e))
+        finally:
+            if fd is not None:
+                self.cfs.close(fd)
+
+    def unlink(self, path) -> None:
+        """
+        Removes a file, link, or symbolic link.
+        :param path: The path of the file or link to unlink.
+        """
+        self.cfs.unlink(path)
+
+    def statfs(self, path) -> dict:
+        """
+        Get the statfs of the specified path by xattr.
+        :param path: The path of the directory/file.
+        :type path: str
+        :return: Returns a dictionary containing 'bytes',
+            'files' and 'subdirs'.
+        :rtype: dict
+        """
+        rbytes = 0
+        rfiles = 0
+        rsubdirs = 0
+        with suppress(cephfs.NoData):
+            rbytes = int(self.cfs.getxattr(path, 'ceph.dir.rbytes'))
+            rfiles = int(self.cfs.getxattr(path, 'ceph.dir.rfiles'))
+            rsubdirs = int(self.cfs.getxattr(path, 'ceph.dir.rsubdirs'))
+        return {'bytes': rbytes, 'files': rfiles, 'subdirs': rsubdirs}