Microbenchmarks improvements and bug fixes

gcsfs/tests/perf/microbenchmarks/conftest.py

@@ -191,7 +191,7 @@ def _benchmark_listing_fixture_helper(
 
             levels[d] = current_level_folders
 
-    # Create empyt folders first if specified
+    # Create empty folders first if specified
     if create_folders:
         logging.info(
             f"Setting up benchmark '{params.name}': creating {len(target_dirs)} "

gcsfs/tests/perf/microbenchmarks/delete/configs.yaml

@@ -5,15 +5,15 @@ common:
 
 scenarios:
   - name: "delete_flat"
-    folders: [256]
+    folders: [1024, 2048, 4096]
     files: [65536, 131072]
 
   - name: "delete_recursive"
     depth: 8
-    folders: [256]
+    folders: [1024, 2048, 4096]
     files: [65536, 131072]
 
   - name: "delete_recursive_deep"
     depth: 24
-    folders: [256]
+    folders: [1024, 2048, 4096]
     files: [65536, 131072]

gcsfs/tests/perf/microbenchmarks/info/configs.py

@@ -6,13 +6,22 @@ class InfoConfigurator(ListingConfigurator):
     param_class = InfoBenchmarkParameters
 
     def _get_folders_list(self, scenario, common_config):
-        return common_config.get("folders", [1])
+        return scenario.get("folders", [1])
 
     def _get_files_list(self, scenario, common_config):
-        return common_config.get("files", [1])
+        return scenario.get("files", [1])
 
     def _get_extra_iterables(self, scenario, common_config):
-        return [scenario.get("target_types", ["bucket", "folder", "file"])]
+        target_types = scenario.get("target_type")
+        if target_types:
+            if isinstance(target_types, str):
+                target_types = [target_types]
+        else:
+            target_types = scenario.get("target_types", ["bucket", "folder", "file"])
+
+        return [
+            target_types,
+        ]
 
     def _create_case_name(
         self,

gcsfs/tests/perf/microbenchmarks/info/configs.yaml

@@ -3,26 +3,72 @@ common:
     - "regional"
     - "zonal"
     - "hns"
-  files:
-    - 100
-  folders:
-    - 100
 
 scenarios:
-  - name: "info"
+  - name: "info_file"
     pattern: "info"
     depth: 1
+    target_type: "file"
+    files: [10000]
+    folders: [256]
 
-  - name: "info_deep"
+  - name: "info_folder"
+    pattern: "info"
+    depth: 1
+    target_type: "folder"
+    files: [65536] # list_objects performance can be measured for standard buckets with high number of files
+    folders: [256]
+
+  - name: "info_bucket"
+    pattern: "info"
+    depth: 1
+    target_type: "bucket"
+    files: [1]
+    folders: [1]
+
+  - name: "info_deep_file"
+    pattern: "info"
+    depth: 10
+    target_type: "file"
+    files: [10000]
+    folders: [256]
+
+  - name: "info_deep_folder"
     pattern: "info"
     depth: 10
+    target_type: "folder"
+    files: [65536]
+    folders: [256]
 
-  - name: "info_multi_process"
+  - name: "info_multi_process_file"
     pattern: "info"
     depth: 1
+    target_type: "file"
+    files: [10000]
+    folders: [256]
+    processes: [4, 8]
+
+  - name: "info_multi_process_folder"
+    pattern: "info"
+    depth: 1
+    target_type: "folder"
+    files: [65536]
+    folders: [256]
+    processes: [4, 8]
+
+
+  - name: "info_multi_process_deep_file"
+    pattern: "info"
+    depth: 10
+    target_type: "file"
+    files: [10000]
+    folders: [256]
     processes: [4, 8]
 
-  - name: "info_multi_process_deep"
+  - name: "info_multi_process_deep_folder"
     pattern: "info"
     depth: 10
+    target_type: "folder"
+    files: [65536]
+    folders: [256]
     processes: [4, 8]

gcsfs/tests/perf/microbenchmarks/info/test_info.py

@@ -25,7 +25,7 @@ def _info_op(gcs, path, pattern="info"):
     except FileNotFoundError:
         pass
     duration_ms = (time.perf_counter() - start_time) * 1000
-    logging.info(f"{pattern.upper()} : {path} - {duration_ms:.2f} ms.")
+    logging.debug(f"{pattern.upper()} : {path} - {duration_ms:.2f} ms.")
 
 
 def _info_ops(gcs, paths, pattern="info"):
@@ -71,12 +71,15 @@ def test_info_multi_threaded(benchmark, gcsfs_benchmark_info, monitor):
 
     paths = _get_target_paths(target_dirs, file_paths, params)
 
+    chunks = _chunk_list(paths, params.threads)
+    args_list = [(gcs, chunks[i], params.pattern) for i in range(params.threads)]
+
     run_multi_threaded(
         benchmark,
         monitor,
         params,
         _info_ops,
-        (gcs, paths, params.pattern),
+        args_list,
         BENCHMARK_GROUP,
     )
 
@@ -123,9 +126,8 @@ def test_info_multi_process(
 ):
     gcs, target_dirs, file_paths, prefix, params = gcsfs_benchmark_info
 
-    chunks = _chunk_list(
-        _get_target_paths(target_dirs, file_paths, params), params.processes
-    )
+    paths = _get_target_paths(target_dirs, file_paths, params)
+    chunks = _chunk_list(paths, params.processes)
 
     def args_builder(gcs_instance, i, shared_arr):
         return (

gcsfs/tests/perf/microbenchmarks/rename/test_rename.py

@@ -41,5 +41,3 @@ def test_rename_recursive(benchmark, gcsfs_benchmark_rename, monitor):
         (gcs, prefix, prefix_renamed),
         BENCHMARK_GROUP,
     )
-    # Adding a sleep of 60 secs here to ensure that deletion works
-    time.sleep(60)

gcsfs/tests/perf/microbenchmarks/test_configs.py

@@ -159,14 +159,16 @@ def test_listing_configurator(mock_config_dependencies):
 
 def test_info_configurator(mock_config_dependencies):
     """Test that InfoConfigurator correctly builds benchmark parameters."""
-    common = {"bucket_types": ["regional"], "files": [100], "folders": [1]}
+    common = {"bucket_types": ["regional"]}
     scenario = {
         "name": "info_test",
         "processes": [1],
         "threads": [1],
         "depth": 0,
         "pattern": "info",
-        "target_types": ["file"],
+        "target_type": "file",
+        "files": [100],
+        "folders": [1],
     }
 
     configurator = InfoConfigurator("dummy")