Handle history cache failures (#4456)

fixes #747

Author: vladak

opengrok-indexer/src/main/java/org/opengrok/indexer/history/GitRepository.java

@@ -465,9 +465,17 @@ public void accept(String sinceRevision, Consumer<BoundaryChangesets.IdWithProgr
              RevWalk walk = new RevWalk(repository)) {
 
             if (sinceRevision != null) {
-                walk.markUninteresting(walk.lookupCommit(repository.resolve(sinceRevision)));
+                ObjectId objId = repository.resolve(sinceRevision);
+                if (objId == null) {
+                    throw new HistoryException("cannot resolve " + sinceRevision);
+                }
+                walk.markUninteresting(walk.lookupCommit(objId));
             }
-            walk.markStart(walk.parseCommit(repository.resolve(Constants.HEAD)));
+            ObjectId objId = repository.resolve(Constants.HEAD);
+            if (objId == null) {
+                throw new HistoryException("cannot resolve HEAD");
+            }
+            walk.markStart(walk.parseCommit(objId));
 
             for (RevCommit commit : walk) {
                 // Do not abbreviate the Id as this could cause AmbiguousObjectException in getHistory().

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

@@ -35,9 +35,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -944,7 +946,7 @@ public void storeHistory(File file, History history) {
         }
     }
 
-    private void createHistoryCache(Repository repository, String sinceRevision) {
+    private void createHistoryCache(Repository repository, String sinceRevision) throws CacheException, HistoryException {
         if (!repository.isHistoryEnabled()) {
             LOGGER.log(Level.INFO,
                     "Skipping history cache creation for {0} and its subdirectories", repository);
@@ -955,26 +957,19 @@ private void createHistoryCache(Repository repository, String sinceRevision) {
             Statistics elapsed = new Statistics();
 
             LOGGER.log(Level.INFO, "Creating history cache for {0}", repository);
-
-            try {
-                repository.createCache(historyCache, sinceRevision);
-            } catch (Exception e) {
-                LOGGER.log(Level.WARNING,
-                        String.format("An error occurred while creating cache for %s", repository), e);
-            }
-
+            repository.createCache(historyCache, sinceRevision);
             elapsed.report(LOGGER, String.format("Done history cache for %s", repository));
         } else {
             LOGGER.log(Level.WARNING,
                     "Skipping creation of history cache for {0}: Missing SCM dependencies?", repository);
         }
     }
 
-    private void createHistoryCacheReal(Collection<Repository> repositories) {
+    private Map<Repository, Optional<Exception>> createHistoryCacheReal(Collection<Repository> repositories) {
         if (repositories.isEmpty()) {
             LOGGER.log(Level.WARNING, "History cache is enabled however the list of repositories is empty. " +
                     "Either specify the repositories in configuration or let the indexer scan them.");
-            return;
+            return Collections.emptyMap();
         }
 
         Statistics elapsed = new Statistics();
@@ -999,32 +994,37 @@ private void createHistoryCacheReal(Collection<Repository> repositories) {
             }
         }
 
-        LOGGER.log(Level.INFO, "Creating history cache for {0} repositories",
-                repos2process.size());
-        final CountDownLatch latch = new CountDownLatch(repos2process.size());
-        for (final Map.Entry<Repository, String> entry : repos2process.entrySet()) {
-            executor.submit(() -> {
-                try {
-                    createHistoryCache(entry.getKey(), entry.getValue());
-                } catch (Exception ex) {
-                    // We want to catch any exception since we are in thread.
-                    LOGGER.log(Level.WARNING, "createHistoryCacheReal() got exception", ex);
-                } finally {
-                    latch.countDown();
-                }
-            });
+        LOGGER.log(Level.INFO, "Creating history cache for {0} repositories", repos2process.size());
+        Map<Repository, Future<Optional<Exception>>> futures = new HashMap<>();
+        try (Progress progress = new Progress(LOGGER, "repository invalidation", repos2process.size())) {
+            for (final Map.Entry<Repository, String> entry : repos2process.entrySet()) {
+                futures.put(entry.getKey(), executor.submit(() -> {
+                    try {
+                        createHistoryCache(entry.getKey(), entry.getValue());
+                    } catch (Exception ex) {    // We want to catch any exception since we are in thread.
+                        LOGGER.log(Level.WARNING,
+                                String.format("failed to create history cache for %s", entry.getKey()), ex);
+                        return Optional.of(ex);
+                    } finally {
+                        progress.increment();
+                    }
+                    return Optional.empty();
+                }));
+            }
         }
 
         /*
          * Wait until the history of all repositories is done. This is necessary
          * since the next phase of generating index will need the history to
          * be ready as it is recorded in Lucene index.
          */
-        try {
-            latch.await();
-        } catch (InterruptedException ex) {
-            LOGGER.log(Level.SEVERE, "latch exception", ex);
-            return;
+        Map<Repository, Optional<Exception>> results = new HashMap<>();
+        for (Map.Entry<Repository, Future<Optional<Exception>>> entry : futures.entrySet()) {
+            try {
+                results.put(entry.getKey(), entry.getValue().get());
+            } catch (InterruptedException | ExecutionException ex) {
+                results.put(entry.getKey(), Optional.of(ex));
+            }
         }
 
         // The cache has been populated. Now, optimize how it is stored on
@@ -1036,6 +1036,8 @@ private void createHistoryCacheReal(Collection<Repository> repositories) {
         }
         elapsed.report(LOGGER, "Done history cache for all repositories", "indexer.history.cache");
         setHistoryIndexDone();
+
+        return results;
     }
 
     /**
@@ -1044,12 +1046,13 @@ private void createHistoryCacheReal(Collection<Repository> repositories) {
      * internal map, e.g. via {@code setRepositories()} or {@code addRepositories()}.
      *
      * @param repositories list of repository paths
+     * @return map of repository to optional exception
      */
-    public void createHistoryCache(Collection<String> repositories) {
+    public Map<Repository, Optional<Exception>> createHistoryCache(Collection<String> repositories) {
         if (!useHistoryCache()) {
-            return;
+            return Collections.emptyMap();
         }
-        createHistoryCacheReal(getReposFromString(repositories));
+        return createHistoryCacheReal(getReposFromString(repositories));
     }
 
     /**
@@ -1161,13 +1164,14 @@ public List<String> removeAnnotationCache(Collection<RepositoryInfo> repositorie
 
     /**
      * Create the history cache for all repositories.
+     * @return map of repository to optional exception
      */
-    public void createHistoryCache() {
+    public Map<Repository, Optional<Exception>> createHistoryCache() {
         if (!useHistoryCache()) {
-            return;
+            return Collections.emptyMap();
         }
 
-        createHistoryCacheReal(repositories.values());
+        return createHistoryCacheReal(repositories.values());
     }
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java

@@ -42,7 +42,9 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -247,23 +249,57 @@ public IndexDatabase(Project project, IndexDownArgsFactory indexDownArgsFactory,
         LIVE_CHECK_FIELDS.add(QueryBuilder.PATH);
     }
 
+    public static void addIndexDatabaseForProject(@Nullable IndexDatabase db, Project project, List<IndexDatabase> dbs,
+                                           Map<Repository, Optional<Exception>> historyCacheResults) throws IOException {
+
+        Map<Repository, Optional<Exception>> projectReposWithException = historyCacheResults.entrySet().
+                stream().
+                filter(e -> e.getValue().isPresent()).
+                filter(e -> project.equals(Project.getProject(e.getKey().getDirectoryNameRelative()))).
+                collect(Collectors.toMap(Entry::getKey, Entry::getValue));
+
+        if (projectReposWithException.isEmpty()) {
+            dbs.add(db != null ? db : new IndexDatabase(project));
+        } else {
+            LOGGER.log(Level.SEVERE, "Failed to create history cache for some repositories of project {0}: {1}",
+                    new Object[]{project, projectReposWithException});
+        }
+    }
+
+    public static void addIndexDatabase(@Nullable IndexDatabase db, List<IndexDatabase> dbs,
+                                        Map<Repository, Optional<Exception>> historyCacheResults) throws IOException {
+
+        Map<Repository, Optional<Exception>> reposWithException = historyCacheResults.entrySet().stream().
+                filter(e -> e.getValue().isPresent()).
+                collect(Collectors.toMap(Entry::getKey, Entry::getValue));
+
+        if (reposWithException.isEmpty()) {
+            dbs.add(db != null ? db : new IndexDatabase());
+        } else {
+            LOGGER.log(Level.SEVERE, "Failed to create history cache for some repositories: {0}",
+                    reposWithException);
+        }
+    }
+
     /**
      * Update the index database for all the projects.
      *
      * @param listener where to signal the changes to the database
+     * @param historyCacheResults map of repository to optional exception
      * @throws IOException if an error occurs
      */
-    static CountDownLatch updateAll(IndexChangedListener listener) throws IOException {
+    static CountDownLatch updateAll(IndexChangedListener listener,
+                                    Map<Repository, Optional<Exception>> historyCacheResults) throws IOException {
 
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
         List<IndexDatabase> dbs = new ArrayList<>();
 
         if (env.hasProjects()) {
             for (Project project : env.getProjectList()) {
-                dbs.add(new IndexDatabase(project));
+                addIndexDatabaseForProject(null, project, dbs, historyCacheResults);
             }
         } else {
-            dbs.add(new IndexDatabase());
+            addIndexDatabase(null, dbs, historyCacheResults);
         }
 
         IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().getIndexerParallelizer();