Configurable StorageConfigInfo for Polaris benchmark (#20)

* initial commit

* triple quotes

* remove hardcoded FIlE check

* readme

---------

Co-authored-by: Eric Maynard <[email protected]>

Author: eric-maynard

benchmarks/README.md

@@ -232,6 +232,7 @@ The data set size can be adjusted as well. Each namespace is associated with an
 
 The diagram below shows sample catalog, namespace and table definition given the following properties:
 
+-   storage-config-info: `{"storageType": "FILE"}`
 -   Default base location: `file:///tmp/polaris`
 -   Number of namespace properties: `100`
 -   Number of columns per table: `999`

benchmarks/src/gatling/resources/benchmark-defaults.conf

@@ -37,6 +37,10 @@ auth {
 
 # Dataset tree structure configuration
 dataset.tree {
+  # JSON to supply as a StorageConfigInfo when creating a catalog
+  # Default: {"storageType": "FILE"}
+  storage-config-info = """{"storageType": "FILE"}"""
+
   # Number of catalogs to create. Only the first catalog (C_0) will contain the test dataset.
   # Default: 1
   num-catalogs = 1

benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/CatalogActions.scala

@@ -62,7 +62,8 @@ case class CatalogActions(
       val catalogName = s"C_$i"
       Map(
         "catalogName" -> catalogName,
-        "defaultBaseLocation" -> s"${dp.defaultBaseLocation}/$catalogName"
+        "defaultBaseLocation" -> s"${dp.defaultBaseLocation}/$catalogName",
+        "storageConfigInfo" -> dp.storageConfigInfo
       )
     }
     .take(dp.numCatalogs)
@@ -90,9 +91,7 @@ case class CatalogActions(
               |    "properties": {
               |      "default-base-location": "#{defaultBaseLocation}"
               |    },
-              |    "storageConfigInfo": {
-              |      "storageType": "FILE"
-              |    }
+              |    "storageConfigInfo": #{storageConfigInfo}
               |  }
               |}""".stripMargin
           )
@@ -117,7 +116,6 @@ case class CatalogActions(
       .check(jsonPath("$.type").is("INTERNAL"))
       .check(jsonPath("$.name").is("#{catalogName}"))
       .check(jsonPath("$.properties.default-base-location").is("#{defaultBaseLocation}"))
-      .check(jsonPath("$.storageConfigInfo.storageType").is("FILE"))
       .check(jsonPath("$.storageConfigInfo.allowedLocations[0]").is("#{defaultBaseLocation}"))
   )
 

benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala

@@ -73,7 +73,8 @@ object BenchmarkConfig {
       dataset.getInt("views-per-namespace"),
       dataset.getInt("max-views"),
       dataset.getInt("columns-per-view"),
-      dataset.getInt("view-properties")
+      dataset.getInt("view-properties"),
+      dataset.getString("storage-config-info")
     )
 
     BenchmarkConfig(connectionParams, workloadParams, datasetParams)

benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/DatasetParameters.scala

@@ -37,6 +37,7 @@ import org.apache.polaris.benchmarks.NAryTreeBuilder
  * @param numViewsMax The maximum number of views to create. If set to -1, all views are created.
  * @param numColumnsPerView The number of columns per view to create.
  * @param numViewProperties The number of view properties to create.
+ * @param storageConfigInfo The JSON to supply when creating a new StorageConfigInfo for a catalog.
  */
 case class DatasetParameters(
     numCatalogs: Int,
@@ -51,7 +52,8 @@ case class DatasetParameters(
     numViewsPerNs: Int,
     numViewsMax: Int,
     numColumnsPerView: Int,
-    numViewProperties: Int
+    numViewProperties: Int,
+    storageConfigInfo: String
 ) {
   val nAryTree: NAryTreeBuilder = NAryTreeBuilder(nsWidth, nsDepth)
   private val maxPossibleTables = nAryTree.numberOfLastLevelElements * numTablesPerNs