bump history cache chunk counts and make it tunable (#3647)

fixes #3585

Author: Vladimir Kotal

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -308,6 +308,8 @@ public final class Configuration {
     private String indexerAuthenticationToken;
     private boolean allowInsecureTokens;
 
+    private int historyChunkCount;
+
     /*
      * types of handling history for remote SCM repositories:
      *  ON - index history and display it in webapp
@@ -1375,6 +1377,14 @@ public void setAllowInsecureTokens(boolean value) {
         this.allowInsecureTokens = value;
     }
 
+    public int getHistoryChunkCount() {
+        return historyChunkCount;
+    }
+
+    public void setHistoryChunkCount(int historyChunkCount) {
+        this.historyChunkCount = historyChunkCount;
+    }
+
     /**
      * Write the current configuration to a file.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -1360,6 +1360,14 @@ public short getContextSurround() {
         return syncReadConfiguration(Configuration::getContextSurround);
     }
 
+    public int getHistoryChunkCount() {
+        return syncReadConfiguration(Configuration::getHistoryChunkCount);
+    }
+
+    public void setHistoryChunkCount(int chunkCount) {
+        syncWriteConfiguration(chunkCount, Configuration::setHistoryChunkCount);
+    }
+
     public Set<String> getDisabledRepositories() {
         return syncReadConfiguration(Configuration::getDisabledRepositories);
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/history/BoundaryChangesets.java

@@ -22,6 +22,8 @@
  */
 package org.opengrok.indexer.history;
 
+import org.jetbrains.annotations.TestOnly;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
 import org.opengrok.indexer.util.Statistics;
 
@@ -47,18 +49,31 @@ public class BoundaryChangesets {
 
     public BoundaryChangesets(RepositoryWithPerPartesHistory repository) {
         this.repository = repository;
-        this.maxCount = repository.getPerPartesCount();
+
+        int globalPerPartesCount = RuntimeEnvironment.getInstance().getHistoryChunkCount();
+        if (globalPerPartesCount > 0) {
+            this.maxCount = globalPerPartesCount;
+        } else {
+            this.maxCount = repository.getPerPartesCount();
+        }
         if (maxCount <= 1) {
             throw new RuntimeException(String.format("per partes count for repository ''%s'' " +
                     "must be stricly greater than 1", repository.getDirectoryName()));
         }
+        LOGGER.log(Level.FINER, "using history cache chunks with {0} entries for repository {1}",
+                new Object[]{this.maxCount, repository});
     }
 
     private void reset() {
         cnt = 0;
         result.clear();
     }
 
+    @TestOnly
+    int getMaxCount() {
+        return maxCount;
+    }
+
     /**
      * @param sinceRevision start revision ID
      * @return immutable list of revision IDs denoting the intervals

opengrok-indexer/src/main/java/org/opengrok/indexer/history/GitRepository.java

@@ -100,7 +100,7 @@ public class GitRepository extends RepositoryWithPerPartesHistory {
     private static final long serialVersionUID = -6126297612958508386L;
 
     public static final int GIT_ABBREV_LEN = 8;
-    public static final int MAX_CHANGESETS = 512;
+    public static final int MAX_CHANGESETS = 65536;
 
     public GitRepository() {
         type = "git";

opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java

@@ -58,7 +58,7 @@ public class MercurialRepository extends RepositoryWithPerPartesHistory {
 
     private static final long serialVersionUID = 1L;
 
-    public static final int MAX_CHANGESETS = 256;
+    public static final int MAX_CHANGESETS = 131072;
 
     /**
      * The property name used to obtain the client command for this repository.

opengrok-indexer/src/test/java/org/opengrok/indexer/history/BoundaryChangesetsTest.java

@@ -25,10 +25,12 @@
 import org.apache.commons.lang3.tuple.ImmutableTriple;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.Mockito;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.util.TestRepository;
 
 import java.io.File;
@@ -37,6 +39,7 @@
 import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -74,6 +77,16 @@ void testInvalidMaxCount(int maxCount) {
         assertThrows(RuntimeException.class, () -> new BoundaryChangesets(gitSpyRepository));
     }
 
+    @Test
+    void testMaxCountConfiguration() {
+        int maxCount = (RuntimeEnvironment.getInstance().getHistoryChunkCount() + 1) * 2;
+        assertNotEquals(0, maxCount);
+        RuntimeEnvironment.getInstance().setHistoryChunkCount(maxCount);
+        int actualCount = new BoundaryChangesets(gitRepository).getMaxCount();
+        RuntimeEnvironment.getInstance().setHistoryChunkCount(0);
+        assertEquals(maxCount, actualCount);
+    }
+
     /**
      * Used to supply test data for testing {@link BoundaryChangesets#getBoundaryChangesetIDs(String)}.
      * @return triplets of (maximum count, start revision, list of expected revisions)