better auto size for druid.server.maxSize if druid.segmentCache.virtualStorage=true

Author: clintropolis

server/src/main/java/org/apache/druid/client/DruidServerConfig.java

@@ -27,6 +27,7 @@
 import org.apache.druid.java.util.common.HumanReadableBytes;
 import org.apache.druid.java.util.common.HumanReadableBytesRange;
 import org.apache.druid.segment.loading.SegmentLoaderConfig;
+import org.apache.druid.utils.RuntimeInfo;
 
 import javax.validation.constraints.NotNull;
 import java.util.Set;
@@ -35,6 +36,9 @@
  */
 public class DruidServerConfig
 {
+  private static final long VIRTUAL_CAPACITY_PER_GIB_OF_HEAP = HumanReadableBytes.parse("15TiB");
+  private static final long ONE_GIB = HumanReadableBytes.parse("1GiB");
+
   @JsonProperty
   @HumanReadableBytesRange(min = 0)
   private HumanReadableBytes maxSize = HumanReadableBytes.ZERO;
@@ -51,21 +55,34 @@ public class DruidServerConfig
                                                          "druid.metadata.storage.connector.password",
                                                          "password", "key", "token", "pwd");
 
+  private final RuntimeInfo runtimeInfo;
   private SegmentLoaderConfig segmentLoaderConfig;
 
   // Guice inject added here to properly bind this dependency into its dependents such as StatusResource
   @Inject
   @JsonCreator
   public DruidServerConfig(
+      @JacksonInject RuntimeInfo runtimeInfo,
       @JacksonInject SegmentLoaderConfig segmentLoaderConfig
   )
   {
     this.segmentLoaderConfig = segmentLoaderConfig;
+    this.runtimeInfo = runtimeInfo;
   }
 
+  /**
+   * The maximum size in bytes a data node advertises itself as having available to serve data. If not set explicitly,
+   * this value is computed. By default this value is computed using {@link SegmentLoaderConfig#getCombinedMaxSize()}
+   * which is the total physical capacity of the segment storage locations. If the data node is using virtual storage
+   * mode, the default is instead computed based on an assumption that ~1G of heap is "plenty" to be able to serve ~50k
+   * segments, using ~300M segment size as the conversion to storage space translates to ~15T of segments per 1G of heap
+   */
   public long getMaxSize()
   {
     if (maxSize.equals(HumanReadableBytes.ZERO)) {
+      if (segmentLoaderConfig.isVirtualStorage()) {
+        return VIRTUAL_CAPACITY_PER_GIB_OF_HEAP * (runtimeInfo.getMaxHeapSizeBytes() / ONE_GIB);
+      }
       return segmentLoaderConfig.getCombinedMaxSize();
     }
     return maxSize.getBytes();

server/src/test/java/org/apache/druid/client/DruidServerConfigTest.java

@@ -28,8 +28,10 @@
 import org.apache.druid.guice.GuiceInjectors;
 import org.apache.druid.initialization.Initialization;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.HumanReadableBytes;
 import org.apache.druid.segment.loading.SegmentLoaderConfig;
 import org.apache.druid.segment.loading.StorageLocationConfig;
+import org.apache.druid.utils.RuntimeInfo;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -83,10 +85,33 @@ public void testCombinedSize()
     final StorageLocationConfig locationConfig2 = new StorageLocationConfig(testSegmentCacheDir2, 20000000000L, null);
     locations.add(locationConfig1);
     locations.add(locationConfig2);
-    DruidServerConfig druidServerConfig = new DruidServerConfig(new SegmentLoaderConfig().withLocations(locations));
+    DruidServerConfig druidServerConfig = new DruidServerConfig(new RuntimeInfo(), new SegmentLoaderConfig().withLocations(locations));
     Assert.assertEquals(30000000000L, druidServerConfig.getMaxSize());
   }
 
+  @Test
+  public void testComputedVirtualSize()
+  {
+    RuntimeInfo runtimeInfo = new RuntimeInfo()
+    {
+      @Override
+      public long getMaxHeapSizeBytes()
+      {
+        return HumanReadableBytes.parse("3GiB");
+      }
+    };
+    SegmentLoaderConfig segmentLoaderConfig = new SegmentLoaderConfig()
+    {
+      @Override
+      public boolean isVirtualStorage()
+      {
+        return true;
+      }
+    };
+    DruidServerConfig druidServerConfig = new DruidServerConfig(runtimeInfo, segmentLoaderConfig);
+    Assert.assertEquals(HumanReadableBytes.parse("45TiB"), druidServerConfig.getMaxSize());
+  }
+
   @Test
   public void testServerMaxSizePrecedence() throws Exception
   {