docs: Update configuration guide to show optional configs (#1524)

Author: andygrove

common/src/main/scala/org/apache/comet/CometConf.scala

@@ -54,7 +54,7 @@ object CometConf extends ShimCometConf {
   /** List of all configs that is used for generating documentation */
   val allConfs = new ListBuffer[ConfigEntry[_]]
 
-  def register(conf: ConfigEntryWithDefault[_]): Unit = {
+  def register(conf: ConfigEntry[_]): Unit = {
     allConfs.append(conf)
   }
 
@@ -746,13 +746,15 @@ private class TypedConfigBuilder[T](
 
   /** Creates a [[ConfigEntry]] that does not have a default value. */
   def createOptional: OptionalConfigEntry[T] = {
-    new OptionalConfigEntry[T](
+    val conf = new OptionalConfigEntry[T](
       parent.key,
       converter,
       stringConverter,
       parent._doc,
       parent._public,
       parent._version)
+    CometConf.register(conf)
+    conf
   }
 
   /** Creates a [[ConfigEntry]] that has a default value. */

docs/source/user-guide/configs.md

@@ -73,6 +73,7 @@ Comet provides the following configuration settings.
 | spark.comet.expression.allowIncompatible | Comet is not currently fully compatible with Spark for all expressions. Set this config to true to allow them anyway. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html). | false |
 | spark.comet.memory.overhead.factor | Fraction of executor memory to be allocated as additional non-heap memory per executor process for Comet. | 0.2 |
 | spark.comet.memory.overhead.min | Minimum amount of additional memory to be allocated per executor process for Comet, in MiB. | 402653184b |
+| spark.comet.memoryOverhead | The amount of additional memory to be allocated per executor process for Comet, in MiB. This config is optional. If this is not specified, it will be set to `spark.comet.memory.overhead.factor` * `spark.executor.memory`. This is memory that accounts for things like Comet native execution, Comet shuffle, etc. | |
 | spark.comet.metrics.updateInterval | The interval in milliseconds to update metrics. If interval is negative, metrics will be updated upon task completion. | 3000 |
 | spark.comet.nativeLoadRequired | Whether to require Comet native library to load successfully when Comet is enabled. If not, Comet will silently fallback to Spark when it fails to load the native lib. Otherwise, an error will be thrown and the Spark job will be aborted. | false |
 | spark.comet.parquet.enable.directBuffer | Whether to use Java direct byte buffer when reading Parquet. | false |

spark/src/main/scala/org/apache/comet/GenerateDocs.scala

@@ -50,7 +50,11 @@ object GenerateDocs {
         w.write("| Config | Description | Default Value |\n".getBytes)
         w.write("|--------|-------------|---------------|\n".getBytes)
         for (conf <- confs) {
-          w.write(s"| ${conf.key} | ${conf.doc.trim} | ${conf.defaultValueString} |\n".getBytes)
+          if (conf.defaultValue.isEmpty) {
+            w.write(s"| ${conf.key} | ${conf.doc.trim} | |\n".getBytes)
+          } else {
+            w.write(s"| ${conf.key} | ${conf.doc.trim} | ${conf.defaultValueString} |\n".getBytes)
+          }
         }
       } else {
         w.write(s"${line.trim}\n".getBytes)