restructure delete functionality in backend to handle directories

Author: DenisaCG

jupyter_drives/manager.py

@@ -399,8 +399,22 @@ async def delete_file(self, drive_name, path):
         try: 
             # eliminate leading and trailing backslashes
             path = path.strip('/')
-            await obs.delete_async(self._content_managers[drive_name]["store"], path)
 
+            # get list of contents with given prefix (path)
+            stream = obs.list(self._content_managers[drive_name]["store"], path, chunk_size=100, return_arrow=True)
+            async for batch in stream:
+                contents_list = pyarrow.record_batch(batch).to_pylist()
+                # delete each object within directory
+                for object in contents_list:
+                    await obs.delete_async(self._content_managers[drive_name]["store"], object["path"])
+            
+            # delete file
+            await obs.delete_async(self._content_managers[drive_name]["store"], path)
+            
+            # when dealing with S3 directory, use helping function to delete remaining directories
+            if self._config.provider == 's3':
+                await self._delete_directories(drive_name, path)
+            
         except Exception as e:
             raise tornado.web.HTTPError(
             status_code= httpx.codes.BAD_REQUEST,
@@ -507,7 +521,7 @@ def _check_object(self, drive_name, path):
 
     def _create_empty_directory(self, drive_name, path):
         """Helping function to create an empty directory, when dealing with S3 buckets.
-        
+
         Args:
             drive_name: name of drive where to create object
             path: path of new object
@@ -524,6 +538,36 @@ def _create_empty_directory(self, drive_name, path):
             reason=f"The following error occured when creating the directory: {e}",
             )
 
+        return
+    
+    async def _delete_directories(self, drive_name, path):
+        """Helping function to delete directories, when dealing with S3 buckets.
+
+        Args:
+            drive_name: name of drive where to create object
+            path: path of new object
+        """
+        try:
+            location = self._content_managers[drive_name]["location"]
+            if location not in self._s3_clients:
+                self._s3_clients[location] = self._s3_session.client('s3', location)
+
+            # delete remaining sub-directories
+            stream = obs.list(self._content_managers[drive_name]["store"], path, chunk_size=100, return_arrow=True)
+            async for batch in stream:
+                contents_list = pyarrow.record_batch(batch).to_pylist()
+                for object in contents_list:
+                    self._s3_clients[location].delete_object(Bucket=drive_name, Key=object["path"]+'/')
+            
+            # delete main directory
+            self._s3_clients[location].delete_object(Bucket=drive_name, Key=path+'/')
+            
+        except Exception as e:
+             raise tornado.web.HTTPError(
+            status_code= httpx.codes.BAD_REQUEST,
+            reason=f"The following error occured when deleting the directory: {e}",
+            )
+
         return 
 
     async def _call_provider(

src/requests.ts

@@ -254,26 +254,7 @@ export async function deleteObjects(
     path: string;
   }
 ) {
-  // get list of contents with given prefix (path)
-  const response = await requestAPI<any>(
-    'drives/' + driveName + '/' + options.path,
-    'GET'
-  );
-
-  // deleting contents of a directory
-  if (response.data.length !== undefined && response.data.length !== 0) {
-    await Promise.all(
-      response.data.map(async (c: any) => {
-        return Private.deleteSingleObject(driveName, c.path);
-      })
-    );
-  }
-  try {
-    // always deleting the object (file or main directory)
-    return Private.deleteSingleObject(driveName, options.path);
-  } catch (error) {
-    // deleting failed if directory didn't exist and was only part of a path
-  }
+  await requestAPI<any>('drives/' + driveName + '/' + options.path, 'DELETE');
 }
 
 /**
@@ -476,20 +457,6 @@ export const countObjectNameAppearances = async (
 };
 
 namespace Private {
-  /**
-   * Helping function for deleting files inside
-   * a directory, in the case of deleting the directory.
-   *
-   * @param driveName
-   * @param objectPath complete path of object to delete
-   */
-  export async function deleteSingleObject(
-    driveName: string,
-    objectPath: string
-  ) {
-    await requestAPI<any>('drives/' + driveName + '/' + objectPath, 'DELETE');
-  }
-
   /**
    * Helping function for renaming files inside
    * a directory, in the case of deleting the directory.